From 2fb4acaaf6b39520a07dda7af4d0defed7963015 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Feb 2022 14:34:46 -0500 Subject: [PATCH 01/29] TD-13338 SELECT statement translate code --- include/util/taoserror.h | 2 + source/libs/function/src/functionMgt.c | 5 +- source/libs/parser/src/parserImpl.c | 88 ++++++++++++++++++++--- source/libs/parser/test/newParserTest.cpp | 31 +++++++- 4 files changed, 113 insertions(+), 13 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index bf6e101633..f2fe3603f2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -455,6 +455,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) //There mustn't be aggregation #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) //ORDER BY item must be the number of a SELECT-list expression #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression +#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression +#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function #ifdef __cplusplus } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 2138e58ccc..78d2feaa83 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -47,10 +47,11 @@ int32_t fmGetHandle(FuncMgtHandle* pHandle) { int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { SFuncMgtService* pService = (SFuncMgtService*)handle; - pFuncId = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); - if (NULL == pFuncId) { + void* pVal = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); + if (NULL == pVal) { return TSDB_CODE_FAILED; } + *pFuncId = *(int32_t*)pVal; if (*pFuncId < 0 || *pFuncId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; } diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 85554d62a6..5c30813215 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -288,6 +288,10 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "ORDER BY item must be the number of a SELECT-list expression"; case TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION: return "Not a GROUP BY expression"; + case TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION: + return "Not SELECTed expression"; + case TSDB_CODE_PAR_NOT_SINGLE_GROUP: + return "Not a single-group group function"; default: return "Unknown error"; } @@ -333,7 +337,7 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST if ('\0' != pCol->dbName[0]) { cmp = strcmp(pCol->dbName, pTable->dbName); } else { - cmp = strcmp(currentDb, pTable->dbName); + cmp = (QUERY_NODE_REAL_TABLE == nodeType(pTable) ? strcmp(currentDb, pTable->dbName) : 0); } if (0 == cmp) { cmp = strcmp(pCol->tableAlias, pTable->tableAlias); @@ -422,9 +426,11 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { static EDealRes translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t nums = taosArrayGetSize(pTables); + bool foundTable = false; for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) { + foundTable = true; if (findAndSetColumn(pCol, pTable)) { break; } @@ -432,6 +438,10 @@ static EDealRes translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* return DEAL_RES_ERROR; } } + if (!foundTable) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pCol->tableAlias); + return DEAL_RES_ERROR; + } return DEAL_RES_CONTINUE; } @@ -651,7 +661,28 @@ static bool isAliasColumn(SColumnNode* pCol) { return ('\0' == pCol->tableAlias[0]); } -static EDealRes doCheckkExprForGroupBy(SNode* pNode, void* pContext) { +static SNodeList* getGroupByList(STranslateContext* pCxt) { + if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct) { + return pCxt->pCurrStmt->pProjectionList; + } + return pCxt->pCurrStmt->pGroupByList; +} + +static SNode* getGroupByNode(SNode* pNode) { + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + return nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0); + } + return pNode; +} + +static int32_t getGroupByErrorCode(STranslateContext* pCxt) { + if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct) { + return TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION; + } + return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; +} + +static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { STranslateContext* pCxt = (STranslateContext*)pContext; if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { return DEAL_RES_CONTINUE; @@ -660,28 +691,64 @@ static EDealRes doCheckkExprForGroupBy(SNode* pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode; - FOREACH(pGroupNode, pCxt->pCurrStmt->pGroupByList) { - if (nodesEqualNode(nodesListGetNode(((SGroupingSetNode*)pGroupNode)->pParameterList, 0), pNode)) { + FOREACH(pGroupNode, getGroupByList(pCxt)) { + if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) { return DEAL_RES_IGNORE_CHILD; } } if (QUERY_NODE_COLUMN == nodeType(pNode)) { - generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); + generateSyntaxErrMsg(pCxt, getGroupByErrorCode(pCxt)); return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNode(pNode, doCheckkExprForGroupBy, pCxt); + nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList) { - nodesWalkList(pList, doCheckkExprForGroupBy, pCxt); + if (NULL == getGroupByList(pCxt)) { + return TSDB_CODE_SUCCESS; + } + nodesWalkList(pList, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } +typedef struct CheckAggColCoexistCxt { + STranslateContext* pTranslateCxt; + bool existAggFunc; + bool existCol; +} CheckAggColCoexistCxt; + +static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { + CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + pCxt->existAggFunc = true; + return DEAL_RES_IGNORE_CHILD; + } + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + pCxt->existCol = true; + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL != pSelect->pGroupByList) { + return TSDB_CODE_SUCCESS; + } + CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; + nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + if (!pSelect->isDistinct) { + nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + } + if (cxt.existAggFunc && cxt.existCol) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_NOT_SINGLE_GROUP); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -809,7 +876,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { } pCxt->currClause = SQL_CLAUSE_ORDER_BY; int32_t code = translateExprList(pCxt, pSelect->pOrderByList); - if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pGroupByList) { + if (TSDB_CODE_SUCCESS == code) { code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); } return code; @@ -822,7 +889,7 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect pCxt->currClause = SQL_CLAUSE_SELECT; code = translateExprList(pCxt, pSelect->pProjectionList); } - if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pGroupByList) { + if (TSDB_CODE_SUCCESS == code) { code = checkExprListForGroupBy(pCxt, pSelect->pProjectionList); } return code; @@ -895,6 +962,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = translateOrderBy(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = checkAggColCoexist(pCxt, pSelect); + } return code; } diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 2414b7c15f..c91427f03f 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -517,7 +517,7 @@ TEST_F(NewParserTest, selectClause) { setDatabase("root", "test"); // GROUP BY clause - bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0"); + bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0"); ASSERT_TRUE(run()); bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); @@ -538,6 +538,16 @@ TEST_F(NewParserTest, selectClause) { bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY 1"); ASSERT_TRUE(run()); + + // DISTINCT clause + bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY c1"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT c1 + 10, c2 FROM t1 WHERE c1 > 0 ORDER BY c1 + 10, c2"); + ASSERT_TRUE(run()); + + bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2"); + ASSERT_TRUE(run()); } TEST_F(NewParserTest, selectSyntaxError) { @@ -573,6 +583,9 @@ TEST_F(NewParserTest, selectSemanticError) { bind("SELECT * FROM test.t10"); ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + bind("SELECT t2.c1 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); + // TSDB_CODE_PAR_AMBIGUOUS_COLUMN bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_AMBIGUOUS_COLUMN)); @@ -608,7 +621,7 @@ TEST_F(NewParserTest, selectSemanticError) { bind("SELECT c2 FROM t1 ORDER BY 2"); ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); - //TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION + // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION bind("SELECT count(*) cnt FROM t1 HAVING c1 > 0"); ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); @@ -620,4 +633,18 @@ TEST_F(NewParserTest, selectSemanticError) { bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c2 > 0 ORDER BY c1"); ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); + + // TSDB_CODE_PAR_NOT_SINGLE_GROUP + bind("SELECT count(*), c1 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + bind("SELECT count(*) FROM t1 ORDER BY c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + bind("SELECT c1 FROM t1 ORDER BY count(*)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); + + // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION + bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } From c524cc8a55afafb3b35b0b4a489d267dcf067325 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Feb 2022 00:40:59 -0500 Subject: [PATCH 02/29] TD-13338 SELECT statement translate code --- source/libs/parser/src/parserImpl.c | 13 +++++++++---- source/libs/parser/test/newParserTest.cpp | 9 +++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 5c30813215..cd42b9ad64 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -661,8 +661,12 @@ static bool isAliasColumn(SColumnNode* pCol) { return ('\0' == pCol->tableAlias[0]); } +static bool isDistinctOrderBy(STranslateContext* pCxt) { + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); +} + static SNodeList* getGroupByList(STranslateContext* pCxt) { - if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct) { + if (isDistinctOrderBy(pCxt)) { return pCxt->pCurrStmt->pProjectionList; } return pCxt->pCurrStmt->pGroupByList; @@ -676,7 +680,7 @@ static SNode* getGroupByNode(SNode* pNode) { } static int32_t getGroupByErrorCode(STranslateContext* pCxt) { - if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct) { + if (isDistinctOrderBy(pCxt)) { return TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION; } return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; @@ -687,7 +691,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode; @@ -696,7 +700,8 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(pNode)) { + if (QUERY_NODE_COLUMN == nodeType(pNode) || + (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && isDistinctOrderBy(pCxt))) { generateSyntaxErrMsg(pCxt, getGroupByErrorCode(pCxt)); return DEAL_RES_ERROR; } diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index c91427f03f..2616c6c251 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -548,6 +548,9 @@ TEST_F(NewParserTest, selectClause) { bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2"); ASSERT_TRUE(run()); + + bind("SELECT DISTINCT count(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 ORDER BY count(c2)"); + ASSERT_TRUE(run()); } TEST_F(NewParserTest, selectSyntaxError) { @@ -647,4 +650,10 @@ TEST_F(NewParserTest, selectSemanticError) { // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts"); ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); + + bind("SELECT DISTINCT c1 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); + + bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } From cdd60902d75f13ed2a010ff1fe6913755592bcaa Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Feb 2022 05:43:14 -0500 Subject: [PATCH 03/29] TD-13338 SELECT statement translate code --- include/libs/function/functionMgt.h | 2 +- include/nodes/nodes.h | 274 ++--------------------- include/nodes/querynodes.h | 280 ++++++++++++++++++++++++ source/libs/parser/inc/astCreateFuncs.h | 2 +- source/libs/parser/inc/parserImpl.h | 3 +- source/libs/planner/inc/plannerImpl.h | 27 +++ source/libs/planner/src/plannerImpl.c | 19 ++ source/nodes/src/nodesEqualFuncs.c | 2 +- source/nodes/src/nodesTraverseFuncs.c | 142 +++++++++++- source/nodes/src/nodesUtilFuncs.c | 2 +- source/nodes/test/nodesTest.cpp | 38 +++- 11 files changed, 520 insertions(+), 271 deletions(-) create mode 100644 include/nodes/querynodes.h create mode 100644 source/libs/planner/inc/plannerImpl.h create mode 100644 source/libs/planner/src/plannerImpl.c diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index bf3a45fcca..c3c8b5c4ce 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "nodes.h" +#include "querynodes.h" typedef enum EFunctionType { // aggregate function diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index b182063808..321190d254 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -40,7 +40,11 @@ extern "C" { (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ cell1 = cell1->pNext, cell2 = cell2->pNext) +#define FOREACH_FOR_REWRITE(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) + typedef enum ENodeType { + // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, VALUE, OPERATOR, FUNCTION and so on. QUERY_NODE_COLUMN = 1, QUERY_NODE_VALUE, QUERY_NODE_OPERATOR, @@ -59,9 +63,10 @@ typedef enum ENodeType { QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, - // only for parser + // Only be used in parser module. QUERY_NODE_RAW_EXPR, + // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, QUERY_NODE_SHOW_STMT @@ -87,248 +92,6 @@ typedef struct SNodeList { SListCell* pTail; } SNodeList; -typedef struct SRawExprNode { - ENodeType nodeType; - char* p; - uint32_t n; - SNode* pNode; -} SRawExprNode; - -typedef struct SDataType { - uint8_t type; - uint8_t precision; - uint8_t scale; - int32_t bytes; -} SDataType; - -typedef struct SExprNode { - ENodeType nodeType; - SDataType resType; - char aliasName[TSDB_COL_NAME_LEN]; - SNodeList* pAssociationList; -} SExprNode; - -typedef enum EColumnType { - COLUMN_TYPE_COLUMN = 1, - COLUMN_TYPE_TAG -} EColumnType; - -typedef struct SColumnNode { - SExprNode node; // QUERY_NODE_COLUMN - int16_t colId; - EColumnType colType; // column or tag - char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; - char tableAlias[TSDB_TABLE_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; - SNode* pProjectRef; -} SColumnNode; - -typedef struct SValueNode { - SExprNode node; // QUERY_NODE_VALUE - char* literal; - bool isDuration; - union { - bool b; - int64_t i; - uint64_t u; - double d; - char* p; - } datum; -} SValueNode; - -typedef enum EOperatorType { - // arithmetic operator - OP_TYPE_ADD = 1, - OP_TYPE_SUB, - OP_TYPE_MULTI, - OP_TYPE_DIV, - OP_TYPE_MOD, - - // comparison operator - OP_TYPE_GREATER_THAN, - OP_TYPE_GREATER_EQUAL, - OP_TYPE_LOWER_THAN, - OP_TYPE_LOWER_EQUAL, - OP_TYPE_EQUAL, - OP_TYPE_NOT_EQUAL, - OP_TYPE_IN, - OP_TYPE_NOT_IN, - OP_TYPE_LIKE, - OP_TYPE_NOT_LIKE, - OP_TYPE_MATCH, - OP_TYPE_NMATCH, - - // json operator - OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS -} EOperatorType; - -typedef struct SOperatorNode { - SExprNode node; // QUERY_NODE_OPERATOR - EOperatorType opType; - SNode* pLeft; - SNode* pRight; -} SOperatorNode; - -typedef enum ELogicConditionType { - LOGIC_COND_TYPE_AND, - LOGIC_COND_TYPE_OR, - LOGIC_COND_TYPE_NOT, -} ELogicConditionType; - -typedef struct SLogicConditionNode { - ENodeType type; // QUERY_NODE_LOGIC_CONDITION - ELogicConditionType condType; - SNodeList* pParameterList; -} SLogicConditionNode; - -typedef struct SIsNullCondNode { - ENodeType type; // QUERY_NODE_IS_NULL_CONDITION - SNode* pExpr; - bool isNull; -} SIsNullCondNode; - -typedef struct SNodeListNode { - ENodeType type; // QUERY_NODE_NODE_LIST - SNodeList* pNodeList; -} SNodeListNode; - -typedef struct SFunctionNode { - SExprNode node; // QUERY_NODE_FUNCTION - char functionName[TSDB_FUNC_NAME_LEN]; - int32_t funcId; - int32_t funcType; - SNodeList* pParameterList; -} SFunctionNode; - -typedef struct STableNode { - ENodeType type; - char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; - char tableAlias[TSDB_TABLE_NAME_LEN]; -} STableNode; - -struct STableMeta; - -typedef struct SRealTableNode { - STableNode table; // QUERY_NODE_REAL_TABLE - struct STableMeta* pMeta; -} SRealTableNode; - -typedef struct STempTableNode { - STableNode table; // QUERY_NODE_TEMP_TABLE - SNode* pSubquery; -} STempTableNode; - -typedef enum EJoinType { - JOIN_TYPE_INNER = 1 -} EJoinType; - -typedef struct SJoinTableNode { - STableNode table; // QUERY_NODE_JOIN_TABLE - EJoinType joinType; - SNode* pLeft; - SNode* pRight; - SNode* pOnCond; -} SJoinTableNode; - -typedef enum EGroupingSetType { - GP_TYPE_NORMAL = 1 -} EGroupingSetType; - -typedef struct SGroupingSetNode { - ENodeType type; // QUERY_NODE_GROUPING_SET - EGroupingSetType groupingSetType; - SNodeList* pParameterList; -} SGroupingSetNode; - -typedef enum EOrder { - ORDER_ASC = 1, - ORDER_DESC -} EOrder; - -typedef enum ENullOrder { - NULL_ORDER_DEFAULT = 1, - NULL_ORDER_FIRST, - NULL_ORDER_LAST -} ENullOrder; - -typedef struct SOrderByExprNode { - ENodeType type; // QUERY_NODE_ORDER_BY_EXPR - SNode* pExpr; - EOrder order; - ENullOrder nullOrder; -} SOrderByExprNode; - -typedef struct SLimitNode { - ENodeType type; // QUERY_NODE_LIMIT - uint64_t limit; - uint64_t offset; -} SLimitNode; - -typedef struct SStateWindowNode { - ENodeType type; // QUERY_NODE_STATE_WINDOW - SNode* pCol; -} SStateWindowNode; - -typedef struct SSessionWindowNode { - ENodeType type; // QUERY_NODE_SESSION_WINDOW - int64_t gap; // gap between two session window(in microseconds) - SNode* pCol; -} SSessionWindowNode; - -typedef struct SIntervalWindowNode { - ENodeType type; // QUERY_NODE_INTERVAL_WINDOW - SNode* pInterval; // SValueNode - SNode* pOffset; // SValueNode - SNode* pSliding; // SValueNode - SNode* pFill; -} SIntervalWindowNode; - -typedef enum EFillMode { - FILL_MODE_NONE = 1, - FILL_MODE_VALUE, - FILL_MODE_PREV, - FILL_MODE_NULL, - FILL_MODE_LINEAR, - FILL_MODE_NEXT -} EFillMode; - -typedef struct SFillNode { - ENodeType type; // QUERY_NODE_FILL - EFillMode mode; - SNode* pValues; // SNodeListNode -} SFillNode; - -typedef struct SSelectStmt { - ENodeType type; // QUERY_NODE_SELECT_STMT - bool isDistinct; - SNodeList* pProjectionList; // SNode - SNode* pFromTable; - SNode* pWhere; - SNodeList* pPartitionByList; // SNode - SNode* pWindow; - SNodeList* pGroupByList; // SGroupingSetNode - SNode* pHaving; - SNodeList* pOrderByList; // SOrderByExprNode - SNode* pLimit; - SNode* pSlimit; -} SSelectStmt; - -typedef enum ESetOperatorType { - SET_OP_TYPE_UNION_ALL = 1 -} ESetOperatorType; - -typedef struct SSetOperator { - ENodeType type; // QUERY_NODE_SET_OPERATOR - ESetOperatorType opType; - SNode* pLeft; - SNode* pRight; - SNodeList* pOrderByList; // SOrderByExprNode - SNode* pLimit; -} SSetOperator; - SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); @@ -343,12 +106,18 @@ typedef enum EDealRes { DEAL_RES_IGNORE_CHILD, DEAL_RES_ERROR, } EDealRes; -typedef EDealRes (*FQueryNodeWalker)(SNode* pNode, void* pContext); -void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); -void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext); -void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext); +typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); +void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext); +void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext); +void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); + +typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); +void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNode* a, const SNode* b); @@ -357,15 +126,6 @@ void nodesCloneNode(const SNode* pNode); int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); -bool nodesIsExprNode(const SNode* pNode); - -bool nodesIsArithmeticOp(const SOperatorNode* pOp); -bool nodesIsComparisonOp(const SOperatorNode* pOp); -bool nodesIsJsonOp(const SOperatorNode* pOp); - -bool nodesIsTimeorderQuery(const SNode* pQuery); -bool nodesIsTimelineQuery(const SNode* pQuery); - #ifdef __cplusplus } #endif diff --git a/include/nodes/querynodes.h b/include/nodes/querynodes.h new file mode 100644 index 0000000000..646ea63c83 --- /dev/null +++ b/include/nodes/querynodes.h @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_NODES_H_ +#define _TD_QUERY_NODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nodes.h" + +typedef struct SRawExprNode { + ENodeType nodeType; + char* p; + uint32_t n; + SNode* pNode; +} SRawExprNode; + +typedef struct SDataType { + uint8_t type; + uint8_t precision; + uint8_t scale; + int32_t bytes; +} SDataType; + +typedef struct SExprNode { + ENodeType nodeType; + SDataType resType; + char aliasName[TSDB_COL_NAME_LEN]; + SNodeList* pAssociationList; +} SExprNode; + +typedef enum EColumnType { + COLUMN_TYPE_COLUMN = 1, + COLUMN_TYPE_TAG +} EColumnType; + +typedef struct SColumnNode { + SExprNode node; // QUERY_NODE_COLUMN + int16_t colId; + EColumnType colType; // column or tag + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + SNode* pProjectRef; +} SColumnNode; + +typedef struct SValueNode { + SExprNode node; // QUERY_NODE_VALUE + char* literal; + bool isDuration; + union { + bool b; + int64_t i; + uint64_t u; + double d; + char* p; + } datum; +} SValueNode; + +typedef enum EOperatorType { + // arithmetic operator + OP_TYPE_ADD = 1, + OP_TYPE_SUB, + OP_TYPE_MULTI, + OP_TYPE_DIV, + OP_TYPE_MOD, + + // comparison operator + OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_EQUAL, + OP_TYPE_LOWER_THAN, + OP_TYPE_LOWER_EQUAL, + OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, + OP_TYPE_IN, + OP_TYPE_NOT_IN, + OP_TYPE_LIKE, + OP_TYPE_NOT_LIKE, + OP_TYPE_MATCH, + OP_TYPE_NMATCH, + + // json operator + OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_CONTAINS +} EOperatorType; + +typedef struct SOperatorNode { + SExprNode node; // QUERY_NODE_OPERATOR + EOperatorType opType; + SNode* pLeft; + SNode* pRight; +} SOperatorNode; + +typedef enum ELogicConditionType { + LOGIC_COND_TYPE_AND, + LOGIC_COND_TYPE_OR, + LOGIC_COND_TYPE_NOT, +} ELogicConditionType; + +typedef struct SLogicConditionNode { + ENodeType type; // QUERY_NODE_LOGIC_CONDITION + ELogicConditionType condType; + SNodeList* pParameterList; +} SLogicConditionNode; + +typedef struct SIsNullCondNode { + ENodeType type; // QUERY_NODE_IS_NULL_CONDITION + SNode* pExpr; + bool isNull; +} SIsNullCondNode; + +typedef struct SNodeListNode { + ENodeType type; // QUERY_NODE_NODE_LIST + SNodeList* pNodeList; +} SNodeListNode; + +typedef struct SFunctionNode { + SExprNode node; // QUERY_NODE_FUNCTION + char functionName[TSDB_FUNC_NAME_LEN]; + int32_t funcId; + int32_t funcType; + SNodeList* pParameterList; +} SFunctionNode; + +typedef struct STableNode { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; +} STableNode; + +struct STableMeta; + +typedef struct SRealTableNode { + STableNode table; // QUERY_NODE_REAL_TABLE + struct STableMeta* pMeta; +} SRealTableNode; + +typedef struct STempTableNode { + STableNode table; // QUERY_NODE_TEMP_TABLE + SNode* pSubquery; +} STempTableNode; + +typedef enum EJoinType { + JOIN_TYPE_INNER = 1 +} EJoinType; + +typedef struct SJoinTableNode { + STableNode table; // QUERY_NODE_JOIN_TABLE + EJoinType joinType; + SNode* pLeft; + SNode* pRight; + SNode* pOnCond; +} SJoinTableNode; + +typedef enum EGroupingSetType { + GP_TYPE_NORMAL = 1 +} EGroupingSetType; + +typedef struct SGroupingSetNode { + ENodeType type; // QUERY_NODE_GROUPING_SET + EGroupingSetType groupingSetType; + SNodeList* pParameterList; +} SGroupingSetNode; + +typedef enum EOrder { + ORDER_ASC = 1, + ORDER_DESC +} EOrder; + +typedef enum ENullOrder { + NULL_ORDER_DEFAULT = 1, + NULL_ORDER_FIRST, + NULL_ORDER_LAST +} ENullOrder; + +typedef struct SOrderByExprNode { + ENodeType type; // QUERY_NODE_ORDER_BY_EXPR + SNode* pExpr; + EOrder order; + ENullOrder nullOrder; +} SOrderByExprNode; + +typedef struct SLimitNode { + ENodeType type; // QUERY_NODE_LIMIT + uint64_t limit; + uint64_t offset; +} SLimitNode; + +typedef struct SStateWindowNode { + ENodeType type; // QUERY_NODE_STATE_WINDOW + SNode* pCol; +} SStateWindowNode; + +typedef struct SSessionWindowNode { + ENodeType type; // QUERY_NODE_SESSION_WINDOW + int64_t gap; // gap between two session window(in microseconds) + SNode* pCol; +} SSessionWindowNode; + +typedef struct SIntervalWindowNode { + ENodeType type; // QUERY_NODE_INTERVAL_WINDOW + SNode* pInterval; // SValueNode + SNode* pOffset; // SValueNode + SNode* pSliding; // SValueNode + SNode* pFill; +} SIntervalWindowNode; + +typedef enum EFillMode { + FILL_MODE_NONE = 1, + FILL_MODE_VALUE, + FILL_MODE_PREV, + FILL_MODE_NULL, + FILL_MODE_LINEAR, + FILL_MODE_NEXT +} EFillMode; + +typedef struct SFillNode { + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode +} SFillNode; + +typedef struct SSelectStmt { + ENodeType type; // QUERY_NODE_SELECT_STMT + bool isDistinct; + SNodeList* pProjectionList; // SNode + SNode* pFromTable; + SNode* pWhere; + SNodeList* pPartitionByList; // SNode + SNode* pWindow; + SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; + SNodeList* pOrderByList; // SOrderByExprNode + SNode* pLimit; + SNode* pSlimit; +} SSelectStmt; + +typedef enum ESetOperatorType { + SET_OP_TYPE_UNION_ALL = 1 +} ESetOperatorType; + +typedef struct SSetOperator { + ENodeType type; // QUERY_NODE_SET_OPERATOR + ESetOperatorType opType; + SNode* pLeft; + SNode* pRight; + SNodeList* pOrderByList; // SOrderByExprNode + SNode* pLimit; +} SSetOperator; + +bool nodesIsExprNode(const SNode* pNode); + +bool nodesIsArithmeticOp(const SOperatorNode* pOp); +bool nodesIsComparisonOp(const SOperatorNode* pOp); +bool nodesIsJsonOp(const SOperatorNode* pOp); + +bool nodesIsTimeorderQuery(const SNode* pQuery); +bool nodesIsTimelineQuery(const SNode* pQuery); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_NODES_H_*/ \ No newline at end of file diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index 5b97a0e0c6..43bc5349e4 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "nodes.h" +#include "querynodes.h" #include "nodesShowStmts.h" #include "astCreateContext.h" #include "ttoken.h" diff --git a/source/libs/parser/inc/parserImpl.h b/source/libs/parser/inc/parserImpl.h index 183075d465..b55060def7 100644 --- a/source/libs/parser/inc/parserImpl.h +++ b/source/libs/parser/inc/parserImpl.h @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" #include "parser.h" #ifndef _TD_AST_CREATE_FUNCS_H_ @@ -25,6 +25,7 @@ extern "C" { typedef struct SQuery { SNode* pRoot; + // todo reslut meta } SQuery; int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery); diff --git a/source/libs/planner/inc/plannerImpl.h b/source/libs/planner/inc/plannerImpl.h new file mode 100644 index 0000000000..050329693c --- /dev/null +++ b/source/libs/planner/inc/plannerImpl.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_PLANNER_IMPL_H_ +#define _TD_PLANNER_IMPL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_PLANNER_IMPL_H_*/ diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c new file mode 100644 index 0000000000..804f673fff --- /dev/null +++ b/source/libs/planner/src/plannerImpl.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +int32_t getPlan(SNode* pRoot, SQueryPlanNode** pQueryPlan) { + +} \ No newline at end of file diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/nodes/src/nodesEqualFuncs.c index 41d1e5b05d..65eb6e9b32 100644 --- a/source/nodes/src/nodesEqualFuncs.c +++ b/source/nodes/src/nodesEqualFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" #define COMPARE_SCALAR_FIELD(fldname) \ do { \ diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/nodes/src/nodesTraverseFuncs.c index 796aab611b..d1ded390db 100644 --- a/source/nodes/src/nodesTraverseFuncs.c +++ b/source/nodes/src/nodesTraverseFuncs.c @@ -13,16 +13,16 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" typedef enum ETraversalOrder { TRAVERSAL_PREORDER = 1, TRAVERSAL_POSTORDER } ETraversalOrder; -static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); +static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext); -static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { +static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) { if (NULL == pNode) { return DEAL_RES_CONTINUE; } @@ -119,7 +119,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker w return res; } -static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { +static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext) { SNode* node; FOREACH(node, pNodeList) { if (DEAL_RES_ERROR == walkNode(node, order, walker, pContext)) { @@ -129,18 +129,144 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNode return DEAL_RES_CONTINUE; } -void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { +void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { +void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext) { +void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); } -void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) { +void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } + +static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRewriter rewriter, void* pContext); + +static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewriter rewriter, void* pContext) { + if (NULL == pRawNode || NULL == *pRawNode) { + return DEAL_RES_CONTINUE; + } + + EDealRes res = DEAL_RES_CONTINUE; + + if (TRAVERSAL_PREORDER == order) { + res = rewriter(pRawNode, pContext); + if (DEAL_RES_CONTINUE != res) { + return res; + } + } + + SNode* pNode = *pRawNode; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: + // these node types with no subnodes + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + res = rewriteNode(&(pOpNode->pLeft), order, rewriter, pContext); + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pOpNode->pRight), order, rewriter, pContext); + } + break; + } + case QUERY_NODE_LOGIC_CONDITION: + res = rewriteList(((SLogicConditionNode*)pNode)->pParameterList, order, rewriter, pContext); + break; + case QUERY_NODE_IS_NULL_CONDITION: + res = rewriteNode(&(((SIsNullCondNode*)pNode)->pExpr), order, rewriter, pContext); + break; + case QUERY_NODE_FUNCTION: + res = rewriteList(((SFunctionNode*)pNode)->pParameterList, order, rewriter, pContext); + break; + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + break; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + res = rewriteNode(&(pJoinTableNode->pLeft), order, rewriter, pContext); + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pJoinTableNode->pRight), order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pJoinTableNode->pOnCond), order, rewriter, pContext); + } + break; + } + case QUERY_NODE_GROUPING_SET: + res = rewriteList(((SGroupingSetNode*)pNode)->pParameterList, order, rewriter, pContext); + break; + case QUERY_NODE_ORDER_BY_EXPR: + res = rewriteNode(&(((SOrderByExprNode*)pNode)->pExpr), order, rewriter, pContext); + break; + case QUERY_NODE_STATE_WINDOW: + res = rewriteNode(&(((SStateWindowNode*)pNode)->pCol), order, rewriter, pContext); + break; + case QUERY_NODE_SESSION_WINDOW: + res = rewriteNode(&(((SSessionWindowNode*)pNode)->pCol), order, rewriter, pContext); + break; + case QUERY_NODE_INTERVAL_WINDOW: { + SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; + res = rewriteNode(&(pInterval->pInterval), order, rewriter, pContext); + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pOffset), order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pSliding), order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pFill), order, rewriter, pContext); + } + break; + } + case QUERY_NODE_NODE_LIST: + res = rewriteList(((SNodeListNode*)pNode)->pNodeList, order, rewriter, pContext); + break; + case QUERY_NODE_FILL: + res = rewriteNode(&(((SFillNode*)pNode)->pValues), order, rewriter, pContext); + break; + case QUERY_NODE_RAW_EXPR: + res = rewriteNode(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); + break; + default: + break; + } + + if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) { + res = rewriter(pRawNode, pContext); + } + + return res; +} + +static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRewriter rewriter, void* pContext) { + SNode** pNode; + FOREACH_FOR_REWRITE(pNode, pNodeList) { + if (DEAL_RES_ERROR == rewriteNode(pNode, order, rewriter, pContext)) { + return DEAL_RES_ERROR; + } + } + return DEAL_RES_CONTINUE; +} + +void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) { + (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext); +} + +void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { + (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext); +} + +void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { + (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext); +} + +void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { + (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext); +} diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index e0e589157c..5d4ddd0e03 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "nodes.h" +#include "querynodes.h" #include "nodesShowStmts.h" #include "taoserror.h" diff --git a/source/nodes/test/nodesTest.cpp b/source/nodes/test/nodesTest.cpp index 7df3cd8b4c..c116faf4ce 100644 --- a/source/nodes/test/nodesTest.cpp +++ b/source/nodes/test/nodesTest.cpp @@ -15,8 +15,44 @@ #include +#include "querynodes.h" + +using namespace std; + +static EDealRes rewriterTest(SNode** pNode, void* pContext) { + EDealRes* pRes = (EDealRes*)pContext; + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + SOperatorNode* pOp = (SOperatorNode*)(*pNode); + if (QUERY_NODE_VALUE != nodeType(pOp->pLeft) || QUERY_NODE_VALUE != nodeType(pOp->pRight)) { + *pRes = DEAL_RES_ERROR; + } + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + string tmp = to_string(stoi(((SValueNode*)(pOp->pLeft))->literal) + stoi(((SValueNode*)(pOp->pRight))->literal)); + pVal->literal = strdup(tmp.c_str()); + nodesDestroyNode(*pNode); + *pNode = (SNode*)pVal; + } + return DEAL_RES_CONTINUE; +} + TEST(NodesTest, traverseTest) { - // todo + SNode* pRoot = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode* pOp = (SOperatorNode*)pRoot; + SOperatorNode* pLeft = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + pLeft->pLeft = nodesMakeNode(QUERY_NODE_VALUE); + ((SValueNode*)(pLeft->pLeft))->literal = strdup("10"); + pLeft->pRight = nodesMakeNode(QUERY_NODE_VALUE); + ((SValueNode*)(pLeft->pRight))->literal = strdup("5"); + pOp->pLeft = (SNode*)pLeft; + pOp->pRight = nodesMakeNode(QUERY_NODE_VALUE); + ((SValueNode*)(pOp->pRight))->literal = strdup("3"); + + EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); + EDealRes res = DEAL_RES_CONTINUE; + nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res); + EXPECT_EQ(res, DEAL_RES_CONTINUE); + EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE); + EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18"); } int main(int argc, char* argv[]) { From 1984e22587d469cd38d414da67d6a7a6e77717c0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Feb 2022 05:47:59 -0500 Subject: [PATCH 04/29] TD-13338 SELECT statement translate code --- source/libs/planner/src/plannerImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index 804f673fff..6acb927db9 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -14,6 +14,6 @@ */ -int32_t getPlan(SNode* pRoot, SQueryPlanNode** pQueryPlan) { +// int32_t getPlan(SNode* pRoot, SQueryPlanNode** pQueryPlan) { -} \ No newline at end of file +// } \ No newline at end of file From 7e3e7fd7868ae3ebfbf869b0e30696dbc804bae2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Feb 2022 05:52:16 -0500 Subject: [PATCH 05/29] TD-13338 SELECT statement translate code --- source/libs/parser/src/parserImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index cd42b9ad64..84486079c3 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -533,7 +533,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; - pVal->datum.i = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); break; } case TSDB_DATA_TYPE_UTINYINT: @@ -563,7 +563,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { int32_t n = strlen(pVal->literal); char* tmp = calloc(1, n); int32_t len = trimStringCopy(pVal->literal, n, tmp); - if (taosParseTime(tmp, &pVal->datum.u, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(tmp, &pVal->datum.i, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { tfree(tmp); generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); return DEAL_RES_ERROR; From 721dec7dccb60810a9a1489e5c60f913c5660452 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Feb 2022 11:15:09 -0500 Subject: [PATCH 06/29] TD-13338 SELECT statement translate code --- include/{ => libs}/nodes/nodes.h | 0 include/{ => libs}/nodes/nodesShowStmts.h | 0 include/{ => libs}/nodes/querynodes.h | 0 source/CMakeLists.txt | 3 +-- source/libs/CMakeLists.txt | 1 + source/{ => libs}/nodes/CMakeLists.txt | 2 +- source/{ => libs}/nodes/src/nodesCloneFuncs.c | 0 source/{ => libs}/nodes/src/nodesCodeFuncs.c | 0 source/{ => libs}/nodes/src/nodesEqualFuncs.c | 0 source/{ => libs}/nodes/src/nodesTraverseFuncs.c | 0 source/{ => libs}/nodes/src/nodesUtilFuncs.c | 0 source/{ => libs}/nodes/test/CMakeLists.txt | 0 source/{ => libs}/nodes/test/nodesTest.cpp | 0 13 files changed, 3 insertions(+), 3 deletions(-) rename include/{ => libs}/nodes/nodes.h (100%) rename include/{ => libs}/nodes/nodesShowStmts.h (100%) rename include/{ => libs}/nodes/querynodes.h (100%) rename source/{ => libs}/nodes/CMakeLists.txt (84%) rename source/{ => libs}/nodes/src/nodesCloneFuncs.c (100%) rename source/{ => libs}/nodes/src/nodesCodeFuncs.c (100%) rename source/{ => libs}/nodes/src/nodesEqualFuncs.c (100%) rename source/{ => libs}/nodes/src/nodesTraverseFuncs.c (100%) rename source/{ => libs}/nodes/src/nodesUtilFuncs.c (100%) rename source/{ => libs}/nodes/test/CMakeLists.txt (100%) rename source/{ => libs}/nodes/test/nodesTest.cpp (100%) diff --git a/include/nodes/nodes.h b/include/libs/nodes/nodes.h similarity index 100% rename from include/nodes/nodes.h rename to include/libs/nodes/nodes.h diff --git a/include/nodes/nodesShowStmts.h b/include/libs/nodes/nodesShowStmts.h similarity index 100% rename from include/nodes/nodesShowStmts.h rename to include/libs/nodes/nodesShowStmts.h diff --git a/include/nodes/querynodes.h b/include/libs/nodes/querynodes.h similarity index 100% rename from include/nodes/querynodes.h rename to include/libs/nodes/querynodes.h diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index fbf045b99c..2833b329a7 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -3,5 +3,4 @@ add_subdirectory(util) add_subdirectory(common) add_subdirectory(libs) add_subdirectory(client) -add_subdirectory(dnode) -add_subdirectory(nodes) \ No newline at end of file +add_subdirectory(dnode) \ No newline at end of file diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 1d23f333b2..049b69991f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -13,3 +13,4 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) +add_subdirectory(nodes) \ No newline at end of file diff --git a/source/nodes/CMakeLists.txt b/source/libs/nodes/CMakeLists.txt similarity index 84% rename from source/nodes/CMakeLists.txt rename to source/libs/nodes/CMakeLists.txt index b30534f3f2..9a826e034c 100644 --- a/source/nodes/CMakeLists.txt +++ b/source/libs/nodes/CMakeLists.txt @@ -2,7 +2,7 @@ aux_source_directory(src NODES_SRC) add_library(nodes STATIC ${NODES_SRC}) target_include_directories( nodes - PUBLIC "${CMAKE_SOURCE_DIR}/include/nodes" + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/nodes" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( diff --git a/source/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c similarity index 100% rename from source/nodes/src/nodesCloneFuncs.c rename to source/libs/nodes/src/nodesCloneFuncs.c diff --git a/source/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c similarity index 100% rename from source/nodes/src/nodesCodeFuncs.c rename to source/libs/nodes/src/nodesCodeFuncs.c diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c similarity index 100% rename from source/nodes/src/nodesEqualFuncs.c rename to source/libs/nodes/src/nodesEqualFuncs.c diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c similarity index 100% rename from source/nodes/src/nodesTraverseFuncs.c rename to source/libs/nodes/src/nodesTraverseFuncs.c diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c similarity index 100% rename from source/nodes/src/nodesUtilFuncs.c rename to source/libs/nodes/src/nodesUtilFuncs.c diff --git a/source/nodes/test/CMakeLists.txt b/source/libs/nodes/test/CMakeLists.txt similarity index 100% rename from source/nodes/test/CMakeLists.txt rename to source/libs/nodes/test/CMakeLists.txt diff --git a/source/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp similarity index 100% rename from source/nodes/test/nodesTest.cpp rename to source/libs/nodes/test/nodesTest.cpp From 5f2dc1401fe382b05c6e92250ae998b432bc4ba2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 11 Feb 2022 19:52:07 +0800 Subject: [PATCH 07/29] feature/qnode --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 43 +- include/libs/qcom/query.h | 6 +- source/client/src/clientHb.c | 10 +- source/dnode/mgmt/impl/src/dndVnodes.c | 3 +- source/dnode/mnode/impl/src/mndStb.c | 1 + source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/vnd/vnodeMain.c | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 3 + source/libs/catalog/inc/catalogInt.h | 79 +- source/libs/catalog/src/catalog.c | 1634 ++++++++++++--------- source/libs/catalog/test/catalogTests.cpp | 53 +- source/libs/parser/src/astValidate.c | 2 +- source/libs/parser/src/dCDAstProcess.c | 2 +- source/libs/qcom/src/querymsg.c | 4 +- 15 files changed, 1094 insertions(+), 749 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 46ad90c14c..bbe4db89dc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1840,3 +1840,4 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p #endif #endif /*_TD_COMMON_TAOS_MSG_H_*/ + \ No newline at end of file diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ab1298785a..b6054c02b4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,7 +30,7 @@ extern "C" { #include "tmsg.h" #include "transport.h" -struct SCatalog; +typedef struct SCatalog SCatalog; enum { CTG_DBG_DB_NUM = 1, @@ -64,6 +64,7 @@ typedef struct SCatalogCfg { typedef struct SSTableMetaVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; uint64_t suid; int16_t sversion; int16_t tversion; @@ -84,7 +85,7 @@ int32_t catalogInit(SCatalogCfg *cfg); * @param catalogHandle (output, NO need to free it) * @return error code */ -int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); /** * Free a cluster's all catalog info, usually it's not necessary, until the application is closing. @@ -92,9 +93,9 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); * @param pCatalog (input, NO more usage) * @return error code */ -void catalogFreeHandle(struct SCatalog* pCatalog); +void catalogFreeHandle(SCatalog* pCatalog); -int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version); /** * Get a DB's all vgroup info. @@ -106,13 +107,13 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); +int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); -int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); +int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); -int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid); +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); /** * Get a table's meta data. @@ -123,7 +124,7 @@ int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, c * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); /** * Get a super table's meta data. @@ -134,13 +135,13 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, cons * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg); +int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); /** - * Force renew a table's local cached meta data. + * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -148,10 +149,10 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); + int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); /** - * Force renew a table's local cached meta data and get the new one. + * Force refresh a table's local cached meta data and get the new one. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) @@ -160,7 +161,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); + int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); @@ -173,7 +174,7 @@ int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); +int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList); /** * Get a table's vgroup from its name's hash value. @@ -184,7 +185,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, * @param vgInfo (output, vgroup info) * @return error code */ -int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); +int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo); /** @@ -196,14 +197,14 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter * @param pRsp (output, response data) * @return error code */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); +int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); +int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); -int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); /** diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 549f36a898..1b70c78af9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -80,16 +80,16 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; -typedef struct SDBVgroupInfo { +typedef struct SDBVgInfo { int32_t vgVersion; int8_t hashMethod; SHashObj *vgHash; //key:vgId, value:SVgroupInfo -} SDBVgroupInfo; +} SDBVgInfo; typedef struct SUseDbOutput { char db[TSDB_DB_FNAME_LEN]; uint64_t dbId; - SDBVgroupInfo *dbVgroup; + SDBVgInfo *dbVgroup; } SUseDbOutput; enum { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3e1af765b0..3f60c5c6e1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -43,7 +43,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { - SDBVgroupInfo vgInfo = {0}; + SDBVgInfo vgInfo = {0}; vgInfo.vgVersion = rsp->vgVersion; vgInfo.hashMethod = rsp->hashMethod; vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -68,10 +68,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } } - code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo); - if (code) { - taosHashCleanup(vgInfo.vgHash); - } + catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo); } if (code) { @@ -94,13 +91,14 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo rsp->numOfColumns = ntohl(rsp->numOfColumns); rsp->suid = be64toh(rsp->suid); + rsp->dbId = be64toh(rsp->dbId); if (rsp->numOfColumns < 0) { schemaNum = 0; tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); - catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid); + catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid); } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index c4d14ef697..dc524ac9f5 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -382,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId}; + SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnode * pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); @@ -594,6 +594,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { vnodeCfg.pDnode = pDnode; vnodeCfg.pTfs = pDnode->pTfs; + vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 33e51da28a..d71d22698c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1469,3 +1469,4 @@ static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + \ No newline at end of file diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7549772613..520a4d026a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -42,6 +42,7 @@ typedef struct STqCfg { typedef struct SVnodeCfg { int32_t vgId; + uint64_t dbId; SDnode *pDnode; STfs *pTfs; uint64_t wsize; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 6bbf3b959d..c748907d6c 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -29,6 +29,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { cfg.vgId = pVnodeCfg->vgId; cfg.pDnode = pVnodeCfg->pDnode; cfg.pTfs = pVnodeCfg->pTfs; + cfg.dbId = pVnodeCfg->dbId; } // Validate options diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index a45a9d5a72..f541834bec 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -93,6 +93,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { if (pTbCfg->type == META_CHILD_TABLE) { pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); if (pStbCfg == NULL) { + code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; } @@ -116,9 +117,11 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols); pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { + code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; } + pTbMetaMsg->dbId = htobe64(pVnode->config.dbId); memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8f24374387..6bfc8d6ca3 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -27,7 +27,7 @@ extern "C" { #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_DB_NUMBER 20 -#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000 +#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 @@ -47,9 +47,19 @@ enum { CTG_RENT_STABLE, }; +enum { + CTG_ACT_UPDATE_VG = 0, + CTG_ACT_UPDATE_TBL, + CTG_ACT_REMOVE_DB, + CTG_ACT_REMOVE_STB, + CTG_ACT_REMOVE_TBL, + CTG_ACT_MAX +}; + typedef struct SCtgDebug { bool lockDebug; bool cacheDebug; + bool apiDebug; uint32_t showCachePeriodSec; } SCtgDebug; @@ -65,7 +75,7 @@ typedef struct SCtgDBCache { SRWLatch vgLock; uint64_t dbId; int8_t deleted; - SDBVgroupInfo *vgInfo; + SDBVgInfo *vgInfo; SCtgTbMetaCache tbCache; } SCtgDBCache; @@ -85,7 +95,6 @@ typedef struct SCtgRentMgmt { typedef struct SCatalog { uint64_t clusterId; - SRWLatch dbLock; SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -109,15 +118,57 @@ typedef struct SCatalogStat { SCtgCacheStat cache; } SCatalogStat; +typedef struct SCtgUpdateVgMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDBVgInfo* dbInfo; +} SCtgUpdateVgMsg; + +typedef struct SCtgUpdateTblMsg { + SCatalog* pCtg; + STableMetaOutput* output; +} SCtgUpdateTblMsg; + +typedef struct SCtgRemoveDBMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; +} SCtgRemoveDBMsg; + +typedef struct SCtgRemoveStbMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; + uint64_t suid; +} SCtgRemoveStbMsg; + +typedef struct SCtgMetaAction { + int32_t act; + void *data; +} SCtgMetaAction; + +typedef struct SCtgQNode { + SCtgMetaAction action; + struct SCtgQNode *next; +} SCtgQNode; + typedef struct SCatalogMgmt { bool exit; SRWLatch lock; + SRWLatch qlock; + SCtgQNode *head; + SCtgQNode *tail; + tsem_t sem; + pthread_t updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; } SCatalogMgmt; typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); +typedef int32_t (*ctgActFunc)(SCtgMetaAction *); #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -130,17 +181,21 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) #define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) +#define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) -#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgError(param, ...) qError("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCatalog, __VA_ARGS__) -#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCatalog, __VA_ARGS__) +#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) +#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST) + +#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__) #define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) #define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiDebug) { qDebug(__VA_ARGS__); } } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 @@ -181,8 +236,8 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define CTG_API_ENTER() do { CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) -#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(__code); } while (0) +#define CTG_API_ENTER() do { CTG_API_DEBUG("enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_API_DEBUG("leave %s", __FUNCTION__); CTG_RET(__code); } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6ecff87a89..1a4b498c58 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -22,6 +22,8 @@ SCatalogMgmt ctgMgmt = {0}; SCtgDebug gCTGDebug = {0}; +ctgActFunc ctgActFuncs[] = {ctgActUpdateVg, ctgActUpdateTbl, ctgActRemoveDB, ctgActRemoveStb, ctgActRemoveTbl}; + int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; } @@ -44,25 +46,25 @@ int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { return num; } -int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { - if (NULL == pCatalog || NULL == pCatalog->dbCache) { +int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { + if (NULL == pCtg || NULL == pCtg->dbCache) { return 0; } switch (type) { case CTG_DBG_DB_NUM: - return (int32_t)taosHashGetSize(pCatalog->dbCache); + return (int32_t)taosHashGetSize(pCtg->dbCache); case CTG_DBG_DB_RENT_NUM: - return ctgDbgGetRentNum(&pCatalog->dbRent); + return ctgDbgGetRentNum(&pCtg->dbRent); case CTG_DBG_STB_RENT_NUM: - return ctgDbgGetRentNum(&pCatalog->stbRent); + return ctgDbgGetRentNum(&pCtg->stbRent); default: break; } SCtgDBCache *dbCache = NULL; int32_t num = 0; - void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { dbCache = (SCtgDBCache *)pIter; switch (type) { @@ -76,7 +78,7 @@ int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type) { ctgError("invalid type:%d", type); break; } - pIter = taosHashIterate(pCatalog->dbCache, pIter); + pIter = taosHashIterate(pCtg->dbCache, pIter); } return num; @@ -108,79 +110,18 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { -void ctgDbgShowClusterCache(struct SCatalog* pCatalog) { - if (NULL == pCatalog) { +void ctgDbgShowClusterCache(SCatalog* pCtg) { + if (NULL == pCtg) { return; } - CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCatalog->clusterId, pCatalog); - CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_META_NUM), - ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCatalog, CTG_DBG_STB_RENT_NUM)); + CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); + CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - ctgDbgShowDBCache(pCatalog->dbCache); + ctgDbgShowDBCache(pCtg->dbCache); } -int32_t ctgInitDBCache(struct SCatalog* pCatalog) { - if (NULL == pCatalog->dbCache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { - taosHashCleanup(cache); - } - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgInitTbMetaCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { - if (NULL == dbCache->tbCache.metaCache) { - if (dbCache->deleted) { - ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - SHashObj *metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == metaCache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.metaCache, NULL, metaCache)) { - taosHashCleanup(metaCache); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgInitStbCache(struct SCatalog* pCatalog, SCtgDBCache *dbCache) { - if (NULL == dbCache->tbCache.stbCache) { - if (dbCache->deleted) { - ctgInfo("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.stbCache, NULL, cache)) { - taosHashCleanup(cache); - } - } - - return TSDB_CODE_SUCCESS; -} - - - void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { return; @@ -214,89 +155,129 @@ void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } +void ctgFreeVgInfo(SDBVgInfo *vgInfo) { + if (NULL == vgInfo) { + return; + } + + if (vgInfo->vgHash) { + taosHashCleanup(vgInfo->vgHash); + vgInfo->vgHash = NULL; + } + + tfree(vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - atomic_store_8(&dbCache->deleted, 1); - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->vgInfo) { - - if (dbCache->vgInfo->vgHash) { - taosHashCleanup(dbCache->vgInfo->vgHash); - dbCache->vgInfo->vgHash = NULL; - } - - tfree(dbCache->vgInfo); - } + ctgFreeVgInfo (dbCache->vgInfo); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); ctgFreeTableMetaCache(&dbCache->tbCache); } -void ctgFreeHandle(struct SCatalog* pCatalog) { - ctgFreeMetaRent(&pCatalog->dbRent); - ctgFreeMetaRent(&pCatalog->stbRent); - if (pCatalog->dbCache) { - void *pIter = taosHashIterate(pCatalog->dbCache, NULL); +void ctgFreeHandle(SCatalog* pCtg) { + ctgFreeMetaRent(&pCtg->dbRent); + ctgFreeMetaRent(&pCtg->stbRent); + + if (pCtg->dbCache) { + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; + atomic_store_8(&dbCache->deleted, 1); + ctgFreeDbCache(dbCache); - - pIter = taosHashIterate(pCatalog->dbCache, pIter); + + pIter = taosHashIterate(pCtg->dbCache, pIter); } - taosHashCleanup(pCatalog->dbCache); + taosHashCleanup(pCtg->dbCache); } - free(pCatalog); + free(pCtg); } -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbFName, SCtgDBCache **dbCache, bool *inCache) { - if (NULL == pCatalog->dbCache) { + +int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_READ, &dbCache->vgLock); + + if (NULL == dbCache->vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->deleted) { + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + return TSDB_CODE_SUCCESS; +} + +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { + taosHashRelease(pCtg->dbCache, dbCache); +} + +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); +} + +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); +} + + +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { + if (NULL == pCtg->dbCache) { + *pCache = NULL; *inCache = false; ctgWarn("empty db cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - SCtgDBCache *cache = NULL; - - while (true) { - cache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - - if (NULL == cache) { - *inCache = false; - ctgWarn("not in db vgroup cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &cache->vgLock); - if (NULL == cache->vgInfo) { - CTG_UNLOCK(CTG_READ, &cache->vgLock); - taosHashRelease(pCatalog->dbCache, cache); - ctgWarn("db cache vgInfo is NULL, dbFName:%s", dbFName); - - continue; - } - - break; + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + *pCache = NULL; + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + ctgAcquireVgInfo(pCtg, dbCache); + if (NULL == dbCache->vgInfo) { + ctgReleaseDBCache(pCtg, dbCache); + + *pCache = NULL; + *inCache = false; + return TSDB_CODE_SUCCESS; } - *dbCache = cache; + *pCache = dbCache; *inCache = true; - ctgDebug("Got db vgroup from cache, dbFName:%s", dbFName); + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { +int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { char *msg = NULL; int32_t msgLen = 0; @@ -318,6 +299,11 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (CTG_DB_NOT_EXIST(rpcRsp.code)) { + ctgDebug("db not exist in mnode, dbFName:%s", input->db); + return rpcRsp.code; + } + ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -331,28 +317,27 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp return TSDB_CODE_SUCCESS; } -int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCatalog->dbCache) { +int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("db not exist in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } - size_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == tbMeta) { - taosHashRelease(pCatalog->dbCache, dbCache); + taosHashRelease(pCtg->dbCache, dbCache); *exist = 0; ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -361,7 +346,7 @@ int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, cha *exist = 1; - taosHashRelease(pCatalog->dbCache, dbCache); + taosHashRelease(pCtg->dbCache, dbCache); ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -369,29 +354,23 @@ int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, cha } -int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { - if (NULL == pCatalog->dbCache) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { + if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); *pTableMeta = NULL; - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, db, strlen(db)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("no db cache, dbFName:%s, tbName:%s", db, pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - if (NULL == dbCache->tbCache.metaCache) { - *exist = 0; - taosHashRelease(pCatalog->dbCache, dbCache); - ctgWarn("empty tbmeta cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgWarn("no db cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -402,8 +381,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if (NULL == *pTableMeta) { *exist = 0; - taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -412,8 +391,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -422,8 +401,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; @@ -431,17 +410,17 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); tfree(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); + int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -450,15 +429,15 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->dbCache) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType) { + if (NULL == pCtg->dbCache) { ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -466,7 +445,7 @@ int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableN char dbName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbName); - SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + SCtgDBCache *dbCache = taosHashAcquire(pCtg->dbCache, dbName, strlen(dbName)); if (NULL == dbCache) { ctgInfo("db not in cache, dbFName:%s", dbName); return TSDB_CODE_SUCCESS; @@ -474,26 +453,29 @@ int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableN CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == pTableMeta) { - ctgWarn("tbmeta not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); - taosHashRelease(pCatalog->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); + taosHashRelease(pCtg->dbCache, dbCache); return TSDB_CODE_SUCCESS; } *tbType = atomic_load_8(&pTableMeta->tableType); - taosHashRelease(dbCache->tbCache.metaCache, dbCache); - taosHashRelease(pCatalog->dbCache, dbCache); + taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + taosHashRelease(pCtg->dbCache, dbCache); ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -539,15 +521,15 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); + return ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -607,12 +589,12 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray** vgroupList) { +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosHashGetSize(vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { @@ -620,23 +602,23 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + void *pIter = taosHashIterate(vgHash, NULL); while (pIter) { vgInfo = pIter; if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(dbInfo->vgHash, pIter); + pIter = taosHashIterate(vgHash, pIter); vgInfo = NULL; } - *vgroupList = vgList; - vgList = NULL; + *pList = vgList; - ctgDebug("Got vg list from DB, vgNum:%d", vgNum); + ctgDebug("Got vgList from cache, vgNum:%d", vgNum); return TSDB_CODE_SUCCESS; @@ -649,7 +631,7 @@ _return: CTG_RET(code); } -int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); @@ -693,7 +675,6 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn CTG_RET(code); } -#if 1 int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; @@ -713,29 +694,6 @@ int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { return 0; } } -#else - -int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { - return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { - return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { - return 1; - } else { - return 0; - } -} - -#endif int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; @@ -932,38 +890,55 @@ int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t si return TSDB_CODE_SUCCESS; } -int32_t ctgAddDBCache(struct SCatalog *pCatalog, const char *dbFName, SCtgDBCache *dbCache) { +int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { int32_t code = 0; - code = taosHashPut(pCatalog->dbCache, dbFName, strlen(dbFName), dbCache, sizeof(SCtgDBCache)); + SCtgDBCache newDBCache = {0}; + newDBCache.dbId = dbId; + + newDBCache.tbCache.metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.metaCache) { + ctgError("taosHashInit %d metaCache failed", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + newDBCache.tbCache.stbCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.stbCache) { + ctgError("taosHashInit %d stbCache failed", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); if (code) { if (HASH_NODE_EXIST(code)) { ctgDebug("db already in cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; + goto _return; } ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - SDbVgVersion vgVersion = {.dbId = dbCache->dbId, .vgVersion = dbCache->vgInfo ? dbCache->vgInfo->vgVersion : -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("dbCache added, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbCache->dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; _return: - ctgFreeDbCache(dbCache); + ctgFreeDbCache(&newDBCache); CTG_RET(code); } -void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache) { +void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { void *pIter = taosHashIterate(cache->stbCache, NULL); @@ -971,7 +946,7 @@ void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache uint64_t *suid = NULL; taosHashGetKey(pIter, (void **)&suid, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCatalog->stbRent, *suid, ctgSTableVersionCompare)) { + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgSTableVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); } @@ -979,232 +954,246 @@ void ctgRemoveAndFreeTableMeta(struct SCatalog* pCatalog, SCtgTbMetaCache *cache } } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - - ctgFreeTableMetaCache(cache); } -int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, SCtgDBCache *dbCache, const char* dbFName) { - if (taosHashRemove(pCatalog->dbCache, dbFName, strlen(dbFName))) { +int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { + atomic_store_8(&dbCache->deleted, 1); + + ctgRemoveStbRent(pCtg, &dbCache->tbCache); + + ctgFreeDbCache(dbCache); + + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - - atomic_store_8(&dbCache->deleted, 1); - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->vgInfo) { - ctgInfo("cleanup db vgInfo, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); - - if (dbCache->vgInfo->vgHash) { - taosHashCleanup(dbCache->vgInfo->vgHash); - } - - tfree(dbCache->vgInfo); - } - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgRemoveAndFreeTableMeta(pCatalog, &dbCache->tbCache); - - ctgInfo("db removed from cache, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); - - CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbCache->dbId, ctgDbVgVersionCompare)); - - ctgDebug("db removed from rent, dbFName:%s, uid:%"PRIx64, dbFName, dbCache->dbId); return TSDB_CODE_SUCCESS; } +int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + SCtgDBCache *dbCache = NULL; + if (acquire) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); + } else { + dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); + } + + if (NULL == dbCache) { + *pCache = NULL; + ctgDebug("db not in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } -int32_t ctgAcquireDBCache(struct SCatalog* pCatalog, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + if (dbCache->deleted) { + if (acquire) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + ctgDebug("db is removing from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); +} + + +int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - - CTG_LOCK(CTG_WRITE, &pCatalog->dbLock); + ctgGetDBCache(pCtg, dbFName, &dbCache); - while (true) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (dbCache) { - // TODO OPEN IT + if (dbCache) { + // TODO OPEN IT #if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } #else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - code = ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName); - taosHashRelease(pCatalog->dbCache, dbCache); - dbCache = NULL; - if (code) { - if (TSDB_CODE_CTG_DB_DROPPED == code) { - continue; - } - - CTG_ERR_JRET(code); - } + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; } - SCtgDBCache newDBCache = {0}; - newDBCache.dbId = dbId; + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } - CTG_ERR_JRET(ctgAddDBCache(pCatalog, dbFName, &newDBCache)); + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { + int32_t code = 0; + SDBVgInfo* dbInfo = *pDbInfo; + + if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } -_return: + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; - if (dbCache) { - taosHashRelease(pCatalog->dbCache, dbCache); + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); + SDBVgInfo *vgInfo = NULL; + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + if (dbCache->vgInfo) { + if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { + ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(dbCache->vgInfo); + } + + dbCache->vgInfo = dbInfo; + + *pDbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWReleaseVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); CTG_RET(code); } +int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { + SCtgTbMetaCache *tbCache = &dbCache->tbCache; -int32_t ctgUpdateTbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { - CTG_LOCK(CTG_READ, &tbCache->metaLock); + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { + origType = orig->tableType; + origSuid = orig->suid; + + if (origType == TSDB_SUPER_TABLE && ((!isStb) || origSuid != meta->suid)) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgSTableVersionCompare); + } + } + + if (isStb) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + } + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { + if (isStb) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - - ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgUpdateStbMetaImpl(struct SCatalog *pCatalog, SCtgTbMetaCache *tbCache, char *dbFName, char *tbName, STableMeta *meta, int32_t metaSize) { - bool newAdded = false; - int32_t code = 0; - SSTableMetaVersion metaRent = {.suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - STableMeta *orig = taosHashAcquire(tbCache->metaCache, tbName, strlen(tbName)); - if (orig) { - if (orig->suid != meta->suid) { - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, db:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - } - - ctgMetaRentRemove(&pCatalog->stbRent, orig->suid, ctgSTableVersionCompare); - } - - taosHashRelease(tbCache->metaCache, orig); + if (!isStb) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, tbCache, dbFName, tbName, meta, metaSize)); - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPutExt(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (isStb && origSuid == meta->suid) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("update stable to cache, suid:%"PRIx64, meta->suid); - - if (newAdded) { - CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - } else { - CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); - } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + ctgDebug("meta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + + SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + return TSDB_CODE_SUCCESS; - -_return: - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_RET(code); } - -int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - CTG_ERR_RET(ctgInitDBCache(pCatalog)); - - CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, output->dbFName, output->dbId, &dbCache)); - - CTG_ERR_JRET(ctgInitTbMetaCache(pCatalog, dbCache)); - CTG_ERR_JRET(ctgInitStbCache(pCatalog, dbCache)); - - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); - } - - if (CTG_IS_META_CTABLE(output->metaType)) { - goto _return; - } - - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); - - if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { - CTG_ERR_JRET(ctgUpdateStbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); - } else { - CTG_ERR_JRET(ctgUpdateTbMetaImpl(pCatalog, &dbCache->tbCache, output->dbFName, output->tbName, output->tbMeta, tbSize)); - } - -_return: - - if (dbCache) { - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); - } - - CTG_RET(code); -} - -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache) { +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; + int32_t code = 0; if (!forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); if (inCache) { return TSDB_CODE_SUCCESS; } - - ctgDebug("failed to get DB vgroupInfo from cache, dbName:%s, load it from mnode, update:%d", dbFName, forceUpdate); } SUseDbOutput DbOut = {0}; @@ -1214,117 +1203,136 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm input.vgVersion = CTG_DEFAULT_INVALID_VERSION; while (true) { - CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbFName, DbOut.dbId, DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); - if (!inCache) { - ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbFName); - continue; + code = ctgUpdateDBVgInfo(pCtg, dbFName, DbOut.dbId, &DbOut.dbVgroup); + if (code && DbOut.dbVgroup) { + *pInfo = DbOut.dbVgroup; + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); + + if (inCache) { + return TSDB_CODE_SUCCESS; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { + *pOutput = malloc(sizeof(STableMetaOutput)); + if (NULL == *pOutput) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*pOutput, output, sizeof(STableMetaOutput)); + + if (output->tbMeta) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + (*pOutput)->tbMeta = malloc(metaSize); + if (NULL == (*pOutput)->tbMeta) { + qError("malloc %d failed", sizeof(STableMetaOutput)); + tfree(*pOutput); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - break; + memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); } return TSDB_CODE_SUCCESS; } -int32_t ctgValidateAndRemoveStbMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid, bool *removed) { - *removed = false; +void ctgPopAction(SCtgMetaAction **action) { + SCtgQNode *orig = ctgMgmt.head; + + SCtgQNode *node = ctgMgmt.head->next; + ctgMgmt.head = ctgMgmt.head->next; + + tfree(orig); - SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == dbCache) { - ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); - return TSDB_CODE_SUCCESS; + *action = &node->action; +} + + +int32_t ctgPushAction(SCtgMetaAction *action) { + SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); + if (NULL == node) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_RET(TSDB_CODE_CTG_MEM_ERROR); } - - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &suid, sizeof(suid))) { - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgInfo("stb not exist in stbCache, may be removed, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, stbName, strlen(stbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - taosHashRelease(pCatalog->dbCache, dbCache); - ctgError("stb not exist in cache, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - taosHashRelease(pCatalog->dbCache, dbCache); + node->action = *action; + + CTG_LOCK(CTG_WRITE, &ctgMgmt.qlock); + ctgMgmt.tail->next = node; + ctgMgmt.tail = node; + CTG_UNLOCK(CTG_WRITE, &ctgMgmt.qlock); + + tsem_post(&ctgMgmt.sem); - *removed = true; - return TSDB_CODE_SUCCESS; } - -int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); - - STableMetaOutput voutput = {0}; - STableMetaOutput moutput = {0}; - STableMetaOutput *output = &voutput; + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + STableMetaOutput moutput = {0}; + STableMetaOutput *output = malloc(sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + if (CTG_IS_STABLE(isSTable)) { ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); - if (CTG_IS_META_NULL(moutput.metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); - } else { - output = &moutput; + if (CTG_IS_META_NULL(output->metaType)) { + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); - if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); - - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { + ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); - voutput.metaType = moutput.metaType; + tfree(output->tbMeta); - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; - moutput.tbMeta = NULL; - } else if (CTG_IS_META_BOTH(voutput.metaType)) { + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, output)); + } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.dbFName, voutput.tbName, &exist)); + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { - SET_META_TYPE_NULL(voutput.metaType); + SET_META_TYPE_NULL(output->metaType); } - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; + tfree(output->tbMeta); + output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { - tfree(voutput.tbMeta); + tfree(output->tbMeta); - SET_META_TYPE_CTABLE(voutput.metaType); + SET_META_TYPE_CTABLE(output->metaType); } } } @@ -1334,46 +1342,282 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, output)); + if (pOutput) { + CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); + } + + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; + SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + return TSDB_CODE_SUCCESS; _return: - tfree(voutput.tbMeta); - tfree(moutput.tbMeta); + tfree(output->tbMeta); + tfree(msg); CTG_RET(code); } -int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; + int32_t code = 0; if (!forceUpdate) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } + + tfree(*pTableMeta); } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { int32_t tbType = 0; - CTG_ERR_RET(ctgGetTableTypeFromCache(pCatalog, pTableName, &tbType)); + CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); CTG_SET_STABLE(isSTable, tbType); } - CTG_ERR_RET(ctgRenewTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, isSTable)); + STableMetaOutput *output = NULL; - CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + if (CTG_IS_META_TABLE(output->metaType)) { + *pTableMeta = output->tbMeta; + goto _return; + } + + if (CTG_IS_META_BOTH(output->metaType)) { + memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + *pTableMeta = output->tbMeta; + goto _return; + } + + if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { + ctgError("invalid metaType:%d", output->metaType); + tfree(output->tbMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + // HANDLE ONLY CHILD TABLE META + + SName stbName = *pTableName; + strcpy(stbName.tname, output->tbName); + + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); if (0 == exist) { - ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", tNameGetTableName(pTableName)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST); + } + + memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + +_return: + + tfree(output); + + CTG_RET(code); +} + + + +int32_t ctgActUpdateVg(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateVgMsg *msg = action->data; + + CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + +_return: + + tfree(msg->dbInfo); + tfree(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveDB(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveDBMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + ctgInfo("db not exist in cache, may be removed, dbFName:%s", msg->dbFName); + goto _return; } + if (dbCache->dbId != msg->dbId) { + ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + ctgReleaseDBCache(msg->pCtg, dbCache); + goto _return; + } + + CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); + +_return: + + tfree(msg); + + CTG_RET(code); +} + + +int32_t ctgActUpdateTbl(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + STableMetaOutput* output = msg->output; + SCtgDBCache *dbCache = NULL; + + if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + } + + if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + } + +_return: + + if (dbCache) { + taosHashRelease(pCtg->dbCache, dbCache); + } + + tfree(msg); + + CTG_RET(code); +} + + +int32_t ctgActRemoveStb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveStbMsg *msg = action->data; + bool removed = false; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgSTableVersionCompare)); + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + +_return: + + tfree(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveTbl(SCtgMetaAction *action) { + +} + + + +void* ctgUpdateThreadFunc(void* param) { + setThreadName("catalog"); + + qInfo("catalog update thread started"); + + CTG_LOCK(CTG_READ, &ctgMgmt.lock); + + while (true) { + tsem_wait(&ctgMgmt.sem); + + if (atomic_load_8(&ctgMgmt.exit)) { + break; + } + + SCtgMetaAction *action = NULL; + ctgPopAction(&action); + + (*ctgActFuncs[action->act])(action); + } + + CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); + + qInfo("catalog update thread stopped"); + + return NULL; +} + + +int32_t ctgStartUpdateThread() { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&ctgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + + pthread_attr_destroy(&thAttr); return TSDB_CODE_SUCCESS; } @@ -1394,7 +1638,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { } if (ctgMgmt.cfg.maxTblCacheNum == 0) { - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; + ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; } if (ctgMgmt.cfg.dbRentSec == 0) { @@ -1406,7 +1650,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { } } else { ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; + ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } @@ -1417,12 +1661,23 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } + CTG_ERR_RET(ctgStartUpdateThread()); + + tsem_init(&ctgMgmt.sem, 0, 0); + + ctgMgmt.head = calloc(1, sizeof(SCtgQNode)); + if (NULL == ctgMgmt.head) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + ctgMgmt.tail = ctgMgmt.head; + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1455,6 +1710,18 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); + clusterCtg->dbCache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == clusterCtg->dbCache) { + qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + SHashObj *metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == metaCache) { + qError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { @@ -1482,97 +1749,81 @@ _return: CTG_RET(code); } -void catalogFreeHandle(struct SCatalog* pCatalog) { - if (NULL == pCatalog) { +void catalogFreeHandle(SCatalog* pCtg) { + if (NULL == pCtg) { return; } - if (taosHashRemove(ctgMgmt.pCluster, &pCatalog->clusterId, sizeof(pCatalog->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCatalog->clusterId); + if (taosHashRemove(ctgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); return; } - uint64_t clusterId = pCatalog->clusterId; + uint64_t clusterId = pCtg->clusterId; - ctgFreeHandle(pCatalog); + ctgFreeHandle(pCtg); ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } -int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { - if (NULL == pCatalog || NULL == dbName || NULL == version) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version) { CTG_API_ENTER(); - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg || NULL == dbFName || NULL == version) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == pCtg->dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("empty db cache, dbName:%s", dbName); + ctgInfo("empty db cache, dbFName:%s", dbFName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgDBCache *db = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == db) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("db not in cache, dbName:%s", dbName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_LOCK(CTG_READ, &db->vgLock); - - if (NULL == db->vgInfo) { - CTG_UNLOCK(CTG_READ, &db->vgLock); + ctgAcquireVgInfo(pCtg, dbCache); + if (NULL == dbCache->vgInfo) { + ctgReleaseDBCache(pCtg, dbCache); *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("db not in cache, dbName:%s", dbName); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = db->vgInfo->vgVersion; - CTG_UNLOCK(CTG_READ, &db->vgLock); - - taosHashRelease(pCatalog->dbCache, db); + *version = dbCache->vgInfo->vgVersion; - ctgDebug("Got db vgVersion from cache, dbName:%s, vgVersion:%d", dbName, *version); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { - if (NULL == pCatalog || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { CTG_API_ENTER(); - SCtgDBCache* dbCache = NULL; - SVgroupInfo *vgInfo = NULL; + if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + SCtgDBCache* dbCache = NULL; int32_t code = 0; SArray *vgList = NULL; - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache)); - - int32_t vgNum = (int32_t)taosHashGetSize(dbCache->vgInfo->vgHash); - vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); - if (NULL == vgList) { - ctgError("taosArrayInit %d failed", vgNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache, &vgInfo)); + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; } - void *pIter = taosHashIterate(dbCache->vgInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - - if (NULL == taosArrayPush(vgList, vgInfo)) { - ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(dbCache->vgInfo->vgHash, pIter); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(dbCache->vgInfo->vgHash, pIter); - vgInfo = NULL; - } + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, &vgList)); *vgroupList = vgList; vgList = NULL; @@ -1580,218 +1831,221 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* _return: if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); } - if (vgList) { - taosArrayDestroy(vgList); - vgList = NULL; + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } CTG_API_LEAVE(code); } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId, SDBVgroupInfo* dbInfo) { - int32_t code = 0; - +int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { CTG_API_ENTER(); + + int32_t code = 0; - if (NULL == pCatalog || NULL == dbFName || NULL == dbInfo) { + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; + SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_ERR_JRET(ctgInitDBCache(pCatalog)); - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_JRET(ctgAcquireDBCache(pCatalog, dbFName, dbId, &dbCache)); - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (dbCache->deleted) { - ctgInfo("db is dropping, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_ERR_JRET(TSDB_CODE_CTG_DB_DROPPED); - } - - if (NULL == dbCache->vgInfo) { - dbCache->vgInfo = dbInfo; - } else { - if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { - ctgInfo("db vgVersion is old, dbFName:%s, vgVersion:%d, current:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); - - goto _return; - } - - if (dbCache->vgInfo->vgHash) { - ctgDebug("cleanup db vgHash, dbFName:%s", dbFName); - taosHashCleanup(dbCache->vgInfo->vgHash); - dbCache->vgInfo->vgHash = NULL; - } - - tfree(dbCache->vgInfo); - dbCache->vgInfo = dbInfo; - } - + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + msg->dbInfo = dbInfo; dbInfo = NULL; - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + action.data = msg; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); - - ctgDebug("dbCache updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + CTG_ERR_JRET(ctgPushAction(&action)); + CTG_API_LEAVE(code); + _return: - if (dbCache) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->dbLock); - } - if (dbInfo) { taosHashCleanup(dbInfo->vgHash); - dbInfo->vgHash = NULL; tfree(dbInfo); } + + tfree(msg); CTG_API_LEAVE(code); } -int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbFName, uint64_t dbId) { +int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { + CTG_API_ENTER(); + int32_t code = 0; - if (NULL == pCatalog || NULL == dbFName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + if (NULL == pCtg || NULL == dbFName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg->dbCache) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); - if (NULL == dbCache) { - ctgInfo("db not exist in dbCache, may be removed, dbFName:%s", dbFName); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - if (dbCache->dbId != dbId) { - ctgInfo("db id already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, dbFName, dbCache->dbId, dbId); - taosHashRelease(pCatalog->dbCache, dbCache); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); - CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbCache, dbFName)); - _return: - - taosHashRelease(pCatalog->dbCache, dbCache); + + tfree(action.data); CTG_API_LEAVE(code); } -int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid) { +int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { + CTG_API_ENTER(); + int32_t code = 0; - bool removed = false; - if (NULL == pCatalog || NULL == dbName || NULL == stbName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - if (NULL == pCatalog->dbCache) { + if (NULL == pCtg->dbCache) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - - CTG_ERR_RET(ctgValidateAndRemoveStbMeta(pCatalog, dbName, stbName, suid, &removed)); - if (!removed) { - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; + SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - - ctgInfo("stb removed from cache, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); - CTG_ERR_JRET(ctgMetaRentRemove(&pCatalog->stbRent, suid, ctgSTableVersionCompare)); - - ctgDebug("stb removed from rent, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + _return: - + + tfree(action.data); + CTG_API_LEAVE(code); } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); } -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); } -int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg) { - STableMetaOutput output = {0}; +int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == rspMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); + } + int32_t code = 0; - CTG_API_ENTER(); + strcpy(output->dbFName, rspMsg->dbFName); + strcpy(output->tbName, rspMsg->tbName); - strcpy(output.dbFName, rspMsg->dbFName); - strcpy(output.tbName, rspMsg->tbName); + output->dbId = rspMsg->dbId; - SET_META_TYPE_TABLE(output.metaType); + SET_META_TYPE_TABLE(output->metaType); - CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output.tbMeta)); + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; + SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + CTG_API_LEAVE(code); + _return: - tfree(output.tbMeta); + tfree(output->tbMeta); + tfree(output); + tfree(msg); CTG_API_LEAVE(code); } -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - CTG_API_LEAVE(ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTransporter, pMgmtEps, pTableName, isSTable, NULL)); } -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); } -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgroupList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - - CTG_API_ENTER(); STableMeta *tbMeta = NULL; int32_t code = 0; @@ -1799,29 +2053,38 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; - *pVgroupList = NULL; + *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCatalog, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbCache)); - // TODO REMOEV THIS .... + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, false, &dbCache, &vgInfo)); + + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; + } + + /* TODO REMOEV THIS .... if (0 == tbMeta->vgId) { SVgroupInfo vgroup = {0}; - catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pTableName, &vgroup); + catalogGetTableHashVgroup(pCtg, pRpc, pMgmtEps, pTableName, &vgroup); tbMeta->vgId = vgroup.vgId; } - // TODO REMOVE THIS .... + // TODO REMOVE THIS ....*/ if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbCache->vgInfo, pVgroupList)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbCache->vgInfo->vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1837,16 +2100,22 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - *pVgroupList = vgList; + *pVgList = vgList; vgList = NULL; } _return: - tfree(tbMeta); if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + tfree(tbMeta); + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } if (vgList) { @@ -1858,37 +2127,42 @@ _return: } -int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); + SCtgDBCache* dbCache = NULL; + int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbCache)); + SDBVgInfo *vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTransporter, pMgmtEps, db, false, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); _return: if (dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - taosHashRelease(pCatalog->dbCache, dbCache); + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + tfree(vgInfo); } CTG_API_LEAVE(code); } -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); + if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + int32_t code = 0; pRsp->pTableMeta = NULL; @@ -1909,7 +2183,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); @@ -1937,40 +2211,42 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { CTG_API_ENTER(); + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + //TODO CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num) { - if (NULL == pCatalog || NULL == stables || NULL == num) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + if (NULL == pCtg || NULL == stables || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num) { - if (NULL == pCatalog || NULL == dbs || NULL == num) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == dbs || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_ENTER(); - - CTG_API_LEAVE(ctgMetaRentGet(&pCatalog->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } void catalogDestroy(void) { + qInfo("start to destroy catalog"); + if (NULL == ctgMgmt.pCluster || atomic_load_8(&ctgMgmt.exit)) { return; } @@ -1979,13 +2255,13 @@ void catalogDestroy(void) { CTG_LOCK(CTG_WRITE, &ctgMgmt.lock); - SCatalog *pCatalog = NULL; + SCatalog *pCtg = NULL; void *pIter = taosHashIterate(ctgMgmt.pCluster, NULL); while (pIter) { - pCatalog = *(SCatalog **)pIter; + pCtg = *(SCatalog **)pIter; - if (pCatalog) { - catalogFreeHandle(pCatalog); + if (pCtg) { + catalogFreeHandle(pCtg); } pIter = taosHashIterate(ctgMgmt.pCluster, pIter); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index d0f98e3c2a..662923141a 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,6 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist); -extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); void ctgTestSetPrepareTableMeta(); @@ -176,11 +175,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(s->name, "tag1s"); } -void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { +void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; - SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo)); + SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo)); dbVgroup->vgVersion = vgVersion++; @@ -612,7 +611,7 @@ void *ctgTestGetDbVgroupThread(void *param) { int32_t n = 0; while (!ctgTestStop) { - code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); if (code) { assert(0); } @@ -635,12 +634,12 @@ void *ctgTestGetDbVgroupThread(void *param) { void *ctgTestSetSameDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); if (code) { assert(0); } @@ -660,12 +659,12 @@ void *ctgTestSetSameDbVgroupThread(void *param) { void *ctgTestSetDiffDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup); if (code) { assert(0); } @@ -716,14 +715,22 @@ void *ctgTestGetCtableMetaThread(void *param) { void *ctgTestSetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; int32_t n = 0; STableMetaOutput output = {0}; ctgTestBuildCTableMetaOutput(&output); + SCtgMetaAction action = {0}; + + action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - code = ctgUpdateTableMetaCache(pCtg, &output); + SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + msg->pCtg = pCtg; + msg->output = output; + action.data = msg; + + code = ctgActUpdateTbl(&action); if (code) { assert(0); } @@ -984,7 +991,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); tableMeta = NULL; - code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); @@ -1174,7 +1181,7 @@ TEST(tableDistVgroup, normalTable) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1206,7 +1213,7 @@ TEST(tableDistVgroup, childTableCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1237,7 +1244,7 @@ TEST(tableDistVgroup, superTableCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1258,7 +1265,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; SArray *vgList = NULL; ctgTestInitLogFile(); @@ -1279,7 +1286,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); @@ -1288,7 +1295,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epset.numOfEps, 3); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1297,7 +1304,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); + code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); ASSERT_EQ(code, 0); code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); @@ -1305,7 +1312,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(vgInfo.vgId, 7); ASSERT_EQ(vgInfo.epset.numOfEps, 2); - code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); @@ -1321,7 +1328,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; @@ -1372,7 +1379,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; @@ -1425,7 +1432,7 @@ TEST(multiThread, ctableMeta) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; @@ -1477,7 +1484,7 @@ TEST(rentTest, allRent) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index a56a6524fc..ffc9f4a3f6 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3647,7 +3647,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) //TODO remove it int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; - int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); + int32_t code = catalogGetTableDistVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 50ae3bfd26..bbe394d67f 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -50,7 +50,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); - int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + int32_t code = catalogGetDBVgInfo(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); if (code != TSDB_CODE_SUCCESS) { terrno = code; return code; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 3e14bfca09..9f5e1b93e3 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -113,9 +113,9 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); + pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); + qError("calloc %d failed", (int32_t)sizeof(SDBVgInfo)); return TSDB_CODE_TSC_OUT_OF_MEMORY; } From b44baf3c20a6210e305740ad719eb4f242b55251 Mon Sep 17 00:00:00 2001 From: dapan Date: Sat, 12 Feb 2022 09:22:04 +0800 Subject: [PATCH 08/29] feature/qnode --- include/common/tmsg.h | 3 +- source/dnode/mnode/impl/src/mndStb.c | 3 +- source/libs/catalog/src/catalog.c | 83 ++++++++++++----------- source/libs/catalog/test/catalogTests.cpp | 8 ++- 4 files changed, 52 insertions(+), 45 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bbe4db89dc..f10d35c146 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1839,5 +1839,4 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } #endif -#endif /*_TD_COMMON_TAOS_MSG_H_*/ - \ No newline at end of file +#endif /*_TD_COMMON_TAOS_MSG_H_*/ \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d71d22698c..585ea3f7a5 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1468,5 +1468,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} - \ No newline at end of file +} \ No newline at end of file diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1a4b498c58..5195a0adf8 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -18,6 +18,12 @@ #include "tname.h" #include "catalogInt.h" +int32_t ctgActUpdateVg(SCtgMetaAction *action); +int32_t ctgActUpdateTbl(SCtgMetaAction *action); +int32_t ctgActRemoveDB(SCtgMetaAction *action); +int32_t ctgActRemoveStb(SCtgMetaAction *action); +int32_t ctgActRemoveTbl(SCtgMetaAction *action); + SCatalogMgmt ctgMgmt = {0}; SCtgDebug gCTGDebug = {0}; @@ -242,6 +248,44 @@ void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { } +int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + SCtgDBCache *dbCache = NULL; + if (acquire) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); + } else { + dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); + } + + if (NULL == dbCache) { + *pCache = NULL; + ctgDebug("db not in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->deleted) { + if (acquire) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + ctgDebug("db is removing from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); +} + + int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { if (NULL == pCtg->dbCache) { *pCache = NULL; @@ -978,43 +1022,6 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - - if (NULL == dbCache) { - *pCache = NULL; - ctgDebug("db not in cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - if (dbCache->deleted) { - if (acquire) { - ctgReleaseDBCache(pCtg, dbCache); - } - - *pCache = NULL; - ctgDebug("db is removing from cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); -} - -int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); -} - int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { int32_t code = 0; @@ -1235,7 +1242,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) int32_t metaSize = CTG_META_SIZE(output->tbMeta); (*pOutput)->tbMeta = malloc(metaSize); if (NULL == (*pOutput)->tbMeta) { - qError("malloc %d failed", sizeof(STableMetaOutput)); + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); tfree(*pOutput); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 662923141a..fc97040ffb 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -33,12 +33,14 @@ #include "tep.h" #include "trpc.h" #include "tvariant.h" +#include "catalogInt.h" namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); +extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); void ctgTestSetPrepareTableMeta(); void ctgTestSetPrepareCTableMeta(); @@ -725,9 +727,9 @@ void *ctgTestSetCtableMetaThread(void *param) { action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); msg->pCtg = pCtg; - msg->output = output; + msg->output = &output; action.data = msg; code = ctgActUpdateTbl(&action); @@ -1077,7 +1079,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - code = catalogRemoveSTableMeta(pCtg, "1.db1", ctgTestSTablename, ctgTestSuid); + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); ASSERT_EQ(code, 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); From 4bc481663a28a90244e3531b432a27a7a90a49ac Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 12 Feb 2022 14:17:08 +0800 Subject: [PATCH 09/29] fix crash DDL --- source/libs/transport/src/transCli.c | 2 +- source/libs/transport/src/transSrv.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 6630cd7d31..9d13ec8225 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -405,7 +405,7 @@ static void clientWrite(SCliConn* pConn) { pConn->secured = 1; // del later pHead = (STransMsgHead*)buf; - pHead->secured = 0; + pHead->secured = 1; msgLen += sizeof(STransUserMsg); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index a731024aba..cc9933f919 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -231,7 +231,7 @@ static void uvHandleReq(SSrvConn* pConn) { p->chandle = NULL; STransMsgHead* pHead = (STransMsgHead*)p->msg; - if (pHead->secured == 0) { + if (pHead->secured == 1) { STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg)); memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); } From b19987ac3a155f23293021a186fb81e052db4303 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Feb 2022 14:21:26 +0800 Subject: [PATCH 10/29] serialize mnode msg --- source/dnode/mnode/impl/src/mndDnode.c | 32 +++--- source/dnode/mnode/impl/src/mndMnode.c | 109 +++++++++++++------ source/dnode/mnode/impl/test/dnode/dnode.cpp | 11 +- source/dnode/mnode/impl/test/mnode/mnode.cpp | 104 +++++++++++------- 4 files changed, 164 insertions(+), 92 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 26f4f58f45..e104048741 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -541,13 +541,13 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) } static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - int32_t code = -1; - SUserObj *pUser = NULL; - SDnodeObj *pDnode = NULL; - SDropDnodeReq dropReq = {0}; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SDnodeObj *pDnode = NULL; + SMDropMnodeReq dropReq = {0}; - if (tDeserializeSDropDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + if (tDeserializeSMCreateDropMnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto DROP_DNODE_OVER; } @@ -591,14 +591,18 @@ DROP_DNODE_OVER: } static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCfgDnodeReq *pCfg = pReq->rpcMsg.pCont; - pCfg->dnodeId = htonl(pCfg->dnodeId); + SMnode *pMnode = pReq->pMnode; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); + SMCfgDnodeReq cfgReq = {0}; + if (tDeserializeSMCfgDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("dnode:%d, failed to config since %s ", pCfg->dnodeId, terrstr()); + mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); return -1; } @@ -606,15 +610,15 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { mndReleaseDnode(pMnode, pDnode); SDCfgDnodeReq *pCfgDnode = rpcMallocCont(sizeof(SDCfgDnodeReq)); - pCfgDnode->dnodeId = htonl(pCfg->dnodeId); - memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN); + pCfgDnode->dnodeId = htonl(cfgReq.dnodeId); + memcpy(pCfgDnode->config, cfgReq.config, TSDB_DNODE_CONFIG_LEN); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pCfgDnode, .contLen = sizeof(SDCfgDnodeReq), .ahandle = pReq->rpcMsg.ahandle}; - mInfo("dnode:%d, app:%p config:%s req send to dnode", pCfg->dnodeId, rpcMsg.ahandle, pCfg->config); + mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config); mndSendReqToDnode(pMnode, &epSet, &rpcMsg); return 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1e0164a572..57b2e22e77 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" #define TSDB_MNODE_VER_NUMBER 1 #define TSDB_MNODE_RESERVE_SIZE 64 @@ -379,40 +381,58 @@ CREATE_MNODE_OVER: } static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMCreateMnodeReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SMnodeObj *pObj = NULL; + SDnodeObj *pDnode = NULL; + SUserObj *pUser = NULL; + SMCreateMnodeReq createReq = {0}; - pCreate->dnodeId = htonl(pCreate->dnodeId); + if (tDeserializeSMCreateDropMnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_MNODE_OVER; + } - mDebug("mnode:%d, start to create", pCreate->dnodeId); + mDebug("mnode:%d, start to create", createReq.dnodeId); - SMnodeObj *pObj = mndAcquireMnode(pMnode, pCreate->dnodeId); + pObj = mndAcquireMnode(pMnode, createReq.dnodeId); if (pObj != NULL) { - mndReleaseMnode(pMnode, pObj); - mError("mnode:%d, mnode already exist", pObj->id); terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; - return -1; + goto CREATE_MNODE_OVER; } else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) { - mError("qnode:%d, failed to create mnode since %s", pCreate->dnodeId, terrstr()); - return -1; + goto CREATE_MNODE_OVER; } - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { - mError("mnode:%d, dnode not exist", pCreate->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + goto CREATE_MNODE_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_MNODE_OVER; + } + + if (mndCheckDropNodeAuth(pUser)) { + goto CREATE_MNODE_OVER; + } + + code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_MNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); return -1; } - int32_t code = mndCreateMnode(pMnode, pReq, pDnode, pCreate); + mndReleaseMnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - return -1; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { @@ -534,32 +554,51 @@ DROP_MNODE_OVER: } static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SMDropMnodeReq *pDrop = pReq->rpcMsg.pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SMnodeObj *pObj = NULL; + SMDropMnodeReq dropReq = {0}; - mDebug("mnode:%d, start to drop", pDrop->dnodeId); + if (tDeserializeSMCreateDropMnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_MNODE_OVER; + } - if (pDrop->dnodeId <= 0) { + mDebug("mnode:%d, start to drop", dropReq.dnodeId); + + if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_SDB_APP_ERROR; - mError("mnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); - return -1; + goto DROP_MNODE_OVER; } - SMnodeObj *pObj = mndAcquireMnode(pMnode, pDrop->dnodeId); + pObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pObj == NULL) { - mError("mnode:%d, not exist", pDrop->dnodeId); - return -1; + goto DROP_MNODE_OVER; } - int32_t code = mndDropMnode(pMnode, pReq, pObj); - if (code != 0) { - mError("mnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_MNODE_OVER; } - sdbRelease(pMnode->pSdb, pObj); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckCreateNodeAuth(pUser)) { + goto DROP_MNODE_OVER; + } + + code = mndDropMnode(pMnode, pReq, pObj); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_MNODE_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("mnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); + } + + mndReleaseMnode(pMnode, pObj); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp) { diff --git a/source/dnode/mnode/impl/test/dnode/dnode.cpp b/source/dnode/mnode/impl/test/dnode/dnode.cpp index 9af0aad1ae..f575556345 100644 --- a/source/dnode/mnode/impl/test/dnode/dnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/dnode.cpp @@ -75,11 +75,14 @@ TEST_F(MndTestDnode, 01_ShowDnode) { } TEST_F(MndTestDnode, 02_ConfigDnode) { - int32_t contLen = sizeof(SMCfgDnodeReq); + SMCfgDnodeReq cfgReq = {0}; + cfgReq.dnodeId = 1; + strcpy(cfgReq.config, "ddebugflag"); + strcpy(cfgReq.value, "131"); - SMCfgDnodeReq* pReq = (SMCfgDnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - strcpy(pReq->config, "ddebugflag 131"); + int32_t contLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCfgDnodeReq(pReq, contLen, &cfgReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONFIG_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp index 208c40e66d..fd8c83e26a 100644 --- a/source/dnode/mnode/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -60,10 +60,12 @@ TEST_F(MndTestMnode, 01_ShowDnode) { TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 1; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -73,10 +75,12 @@ TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { TEST_F(MndTestMnode, 03_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -107,10 +111,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { { // create mnode - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -134,10 +140,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { { // drop mnode - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -156,10 +164,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { { // drop mnode - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -170,10 +180,12 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -183,10 +195,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { { // continue send message, mnode is creating - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -195,10 +209,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { { // continue send message, mnode is creating - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -214,10 +230,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -232,10 +250,12 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { { // send message first, then dnode2 crash, result is returned, and rollback is started - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); @@ -245,10 +265,12 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { { // continue send message, mnode is dropping - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -257,10 +279,12 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { { // continue send message, mnode is dropping - int32_t contLen = sizeof(SMDropMnodeReq); + SMDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SMDropMnodeReq* pReq = (SMDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -276,10 +300,12 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { - int32_t contLen = sizeof(SMCreateMnodeReq); + SMCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; - SMCreateMnodeReq* pReq = (SMCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); From 3d396ca7ff49307008df2c9fccff70e604d696b2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Feb 2022 16:28:50 +0800 Subject: [PATCH 11/29] serialize func msg --- include/common/tmsg.h | 27 +- source/common/src/tmsg.c | 211 ++++++++++- source/dnode/mnode/impl/inc/mndAuth.h | 5 +- source/dnode/mnode/impl/src/mndAuth.c | 11 +- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndDnode.c | 6 +- source/dnode/mnode/impl/src/mndFunc.c | 234 +++++++----- source/dnode/mnode/impl/src/mndMnode.c | 5 +- source/dnode/mnode/impl/src/mndQnode.c | 6 +- source/dnode/mnode/impl/src/mndSnode.c | 5 +- source/dnode/mnode/impl/test/func/func.cpp | 419 +++++++++++---------- 11 files changed, 594 insertions(+), 339 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c05bbff050..9e99d6a9ba 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -394,8 +394,8 @@ typedef struct { SHashObj* writeDbs; } SGetUserAuthRsp; -int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pReq); -int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pReq); +int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); +int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); typedef struct { int16_t colId; // column id @@ -616,22 +616,31 @@ typedef struct { int64_t signature; int32_t commentSize; int32_t codeSize; - char pCont[]; + char pComment[TSDB_FUNC_COMMENT_LEN]; + char pCode[TSDB_FUNC_CODE_LEN]; } SCreateFuncReq; +int32_t tSerializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); +int32_t tDeserializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); + typedef struct { char name[TSDB_FUNC_NAME_LEN]; int8_t igNotExists; } SDropFuncReq; +int32_t tSerializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq); +int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq); + typedef struct { int32_t numOfFuncs; - char pFuncNames[]; + SArray* pFuncNames; } SRetrieveFuncReq; +int32_t tSerializeSRetrieveFuncReq(void* buf, int32_t bufLen, SRetrieveFuncReq* pReq); +int32_t tDeserializeSRetrieveFuncReq(void* buf, int32_t bufLen, SRetrieveFuncReq* pReq); + typedef struct { char name[TSDB_FUNC_NAME_LEN]; - int8_t align; int8_t funcType; int8_t scriptType; int8_t outputType; @@ -640,14 +649,18 @@ typedef struct { int64_t signature; int32_t commentSize; int32_t codeSize; - char pCont[]; + char pComment[TSDB_FUNC_COMMENT_LEN]; + char pCode[TSDB_FUNC_CODE_LEN]; } SFuncInfo; typedef struct { int32_t numOfFuncs; - char pFuncInfos[]; + SArray* pFuncInfos; } SRetrieveFuncRsp; +int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); +int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); + typedef struct { int32_t statusInterval; int64_t checkTime; // 1970-01-01 00:00:00.000 diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 65b9252fbb..c6096ac4b8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -753,29 +753,29 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * return 0; } -int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pReq) { +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeI8(&encoder, pReq->superAuth) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; + if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; - int32_t numOfReadDbs = taosHashGetSize(pReq->readDbs); - int32_t numOfWriteDbs = taosHashGetSize(pReq->writeDbs); + int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); + int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; - char *db = taosHashIterate(pReq->readDbs, NULL); + char *db = taosHashIterate(pRsp->readDbs, NULL); while (db != NULL) { if (tEncodeCStr(&encoder, db) < 0) return -1; - db = taosHashIterate(pReq->readDbs, db); + db = taosHashIterate(pRsp->readDbs, db); } - db = taosHashIterate(pReq->writeDbs, NULL); + db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { if (tEncodeCStr(&encoder, db) < 0) return -1; - db = taosHashIterate(pReq->writeDbs, db); + db = taosHashIterate(pRsp->writeDbs, db); } tEndEncode(&encoder); @@ -785,10 +785,10 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR return tlen; } -int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pReq) { - pReq->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pReq->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - if (pReq->readDbs == NULL || pReq->writeDbs == NULL) { +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { return -1; } @@ -796,8 +796,8 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->superAuth) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; + if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -808,14 +808,14 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(&decoder, db) < 0) return -1; int32_t len = strlen(db) + 1; - taosHashPut(pReq->readDbs, db, len, db, len); + taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(&decoder, db) < 0) return -1; int32_t len = strlen(db) + 1; - taosHashPut(pReq->writeDbs, db, len, db, len); + taosHashPut(pRsp->writeDbs, db, len, db, len); } tEndDecode(&decoder); @@ -920,3 +920,180 @@ int32_t tDeserializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq * tCoderClear(&decoder); return 0; } + +int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->funcType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->scriptType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->outputType) < 0) return -1; + if (tEncodeI32(&encoder, pReq->outputLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->bufSize) < 0) return -1; + if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commentSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->codeSize) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->funcType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->scriptType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->outputType) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->outputLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->bufSize) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commentSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->codeSize) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->pCode) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropFuncReq(void *buf, int32_t bufLen, SDropFuncReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropFuncReq(void *buf, int32_t bufLen, SDropFuncReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1; + + if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1; + for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { + char *fname = taosArrayGet(pReq->pFuncNames, i); + if (tEncodeCStr(&encoder, fname) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1; + + pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN); + if (pReq->pFuncNames == NULL) return -1; + + for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { + char fname[TSDB_FUNC_NAME_LEN] = {0}; + if (tDecodeCStrTo(&decoder, fname) < 0) return -1; + taosArrayPush(pReq->pFuncNames, fname); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfFuncs) < 0) return -1; + + if (pRsp->numOfFuncs != (int32_t)taosArrayGetSize(pRsp->pFuncInfos)) return -1; + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i); + + if (tEncodeCStr(&encoder, pInfo->name) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->funcType) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->scriptType) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->outputType) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->outputLen) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->bufSize) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; + if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfFuncs) < 0) return -1; + + pRsp->pFuncInfos = taosArrayInit(pRsp->numOfFuncs, sizeof(SFuncInfo)); + if (pRsp->pFuncInfos == NULL) return -1; + + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncInfo fInfo = {0}; + if (tDecodeCStrTo(&decoder, fInfo.name) < 0) return -1; + if (tDecodeI8(&decoder, &fInfo.funcType) < 0) return -1; + if (tDecodeI8(&decoder, &fInfo.scriptType) < 0) return -1; + if (tDecodeI8(&decoder, &fInfo.outputType) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.outputLen) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.bufSize) < 0) return -1; + if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1; + if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; + if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; + taosArrayPush(pRsp->pFuncInfos, &fInfo); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index 65b8642572..ca196f69be 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -29,9 +29,8 @@ int32_t mndCheckCreateUserAuth(SUserObj *pOperUser); int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, SAlterUserReq *pAlter); int32_t mndCheckDropUserAuth(SUserObj *pOperUser); -int32_t mndCheckCreateNodeAuth(SUserObj *pOperUser); -int32_t mndCheckDropNodeAuth(SUserObj *pOperUser); -int32_t mndCheckAlterNodeAuth(SUserObj *pOperUser); +int32_t mndCheckOperateNodeAuth(SUserObj *pOperUser); +int32_t mndCheckOperateFuncAuth(SUserObj *pOperUser); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index f4e385bbf6..14dc451686 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -111,7 +111,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { return -1; } -int32_t mndCheckCreateNodeAuth(SUserObj *pOperUser) { +int32_t mndCheckOperateNodeAuth(SUserObj *pOperUser) { if (pOperUser->superUser) { return 0; } @@ -120,6 +120,11 @@ int32_t mndCheckCreateNodeAuth(SUserObj *pOperUser) { return -1; } -int32_t mndCheckDropNodeAuth(SUserObj *pOperUser) { return mndCheckCreateNodeAuth(pOperUser); } +int32_t mndCheckOperateFuncAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) { + return 0; + } -int32_t mndCheckAlterNodeAuth(SUserObj *pOperUser) { return mndCheckCreateNodeAuth(pOperUser); } + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 668fe27d2c..7bffc482bb 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -293,7 +293,7 @@ static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { goto CREATE_BNODE_OVER; } - if (mndCheckDropNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto CREATE_BNODE_OVER; } @@ -400,7 +400,7 @@ static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { goto DROP_BNODE_OVER; } - if (mndCheckCreateNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto DROP_BNODE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e104048741..3f4213064a 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -496,7 +496,7 @@ static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { goto CREATE_DNODE_OVER; } - if (mndCheckDropNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto CREATE_DNODE_OVER; } @@ -506,7 +506,6 @@ static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { CREATE_DNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); - return -1; } mndReleaseDnode(pMnode, pDnode); @@ -571,7 +570,7 @@ static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { goto DROP_DNODE_OVER; } - if (mndCheckCreateNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto DROP_DNODE_OVER; } @@ -581,7 +580,6 @@ static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { DROP_DNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); - return -1; } mndReleaseDnode(pMnode, pDnode); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b3fc0e1da0..cd5eaca638 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -15,9 +15,11 @@ #define _DEFAULT_SOURCE #include "mndFunc.h" +#include "mndAuth.h" #include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" +#include "mndUser.h" #define SDB_FUNC_VER 1 #define SDB_FUNC_RESERVE_SIZE 64 @@ -201,8 +203,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pC goto CREATE_FUNC_OVER; } - memcpy(func.pComment, pCreate->pCont, pCreate->commentSize); - memcpy(func.pCode, pCreate->pCont + pCreate->commentSize, func.codeSize); + memcpy(func.pComment, pCreate->pComment, pCreate->commentSize); + memcpy(func.pCode, pCreate->pCode, func.codeSize); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_FUNC_OVER; @@ -261,164 +263,202 @@ DROP_FUNC_OVER: } static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SFuncObj *pFunc = NULL; + SCreateFuncReq createReq = {0}; - SCreateFuncReq *pCreate = pReq->rpcMsg.pCont; - pCreate->outputLen = htonl(pCreate->outputLen); - pCreate->bufSize = htonl(pCreate->bufSize); - pCreate->signature = htobe64(pCreate->signature); - pCreate->commentSize = htonl(pCreate->commentSize); - pCreate->codeSize = htonl(pCreate->codeSize); + if (tDeserializeSCreateFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_FUNC_OVER; + } - mDebug("func:%s, start to create", pCreate->name); + mDebug("func:%s, start to create", createReq.name); - SFuncObj *pFunc = mndAcquireFunc(pMnode, pCreate->name); + pFunc = mndAcquireFunc(pMnode, createReq.name); if (pFunc != NULL) { - mndReleaseFunc(pMnode, pFunc); - if (pCreate->igExists) { - mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); - return 0; + if (createReq.igExists) { + mDebug("func:%s, already exist, ignore exist is set", createReq.name); + code = 0; + goto CREATE_FUNC_OVER; } else { terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { - mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->name[0] == 0) { + if (createReq.name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->commentSize <= 0 || pCreate->commentSize > TSDB_FUNC_COMMENT_LEN) { + if (createReq.commentSize <= 0 || createReq.commentSize > TSDB_FUNC_COMMENT_LEN) { terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->codeSize <= 0 || pCreate->codeSize > TSDB_FUNC_CODE_LEN) { + if (createReq.codeSize <= 0 || createReq.codeSize > TSDB_FUNC_CODE_LEN) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->pCont[0] == 0) { + if (createReq.pCode[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - if (pCreate->bufSize <= 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto CREATE_FUNC_OVER; } - int32_t code = mndCreateFunc(pMnode, pReq, pCreate); - if (code != 0) { - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto CREATE_FUNC_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckOperateFuncAuth(pUser)) { + goto CREATE_FUNC_OVER; + } + + code = mndCreateFunc(pMnode, pReq, &createReq); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_FUNC_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("func:%s, failed to create since %s", createReq.name, terrstr()); + } + + mndReleaseFunc(pMnode, pFunc); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SDropFuncReq *pDrop = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SFuncObj *pFunc = NULL; + SDropFuncReq dropReq = {0}; - mDebug("func:%s, start to drop", pDrop->name); - - if (pDrop->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + if (tDeserializeSDropFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_FUNC_OVER; } - SFuncObj *pFunc = mndAcquireFunc(pMnode, pDrop->name); + mDebug("func:%s, start to drop", dropReq.name); + + if (dropReq.name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + goto DROP_FUNC_OVER; + } + + pFunc = mndAcquireFunc(pMnode, dropReq.name); if (pFunc == NULL) { - if (pDrop->igNotExists) { - mDebug("func:%s, not exist, ignore not exist is set", pDrop->name); - return 0; + if (dropReq.igNotExists) { + mDebug("func:%s, not exist, ignore not exist is set", dropReq.name); + code = 0; + goto DROP_FUNC_OVER; } else { terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + goto DROP_FUNC_OVER; } } - int32_t code = mndDropFunc(pMnode, pReq, pFunc); - mndReleaseFunc(pMnode, pFunc); - - if (code != 0) { - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto DROP_FUNC_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckOperateFuncAuth(pUser)) { + goto DROP_FUNC_OVER; + } + + code = mndDropFunc(pMnode, pReq, pFunc); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_FUNC_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); + } + + mndReleaseFunc(pMnode, pFunc); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { - int32_t code = -1; - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SRetrieveFuncReq retrieveReq = {0}; + SRetrieveFuncRsp retrieveRsp = {0}; - SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; - pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); - if (pRetrieve->numOfFuncs <= 0 || pRetrieve->numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { + if (tDeserializeSRetrieveFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto RETRIEVE_FUNC_OVER; + } + + if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; - return -1; + goto RETRIEVE_FUNC_OVER; } - int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN; - int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs; - - SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); - if (pRetrieveRsp == NULL) { + retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs; + retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo)); + if (retrieveRsp.pFuncInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FUNC_RETRIEVE_OVER; + goto RETRIEVE_FUNC_OVER; } - pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); - char *pOutput = pRetrieveRsp->pFuncInfos; - - for (int32_t i = 0; i < pRetrieve->numOfFuncs; ++i) { - char funcName[TSDB_FUNC_NAME_LEN] = {0}; - memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); + for (int32_t i = 0; i < retrieveReq.numOfFuncs; ++i) { + char *funcName = taosArrayGet(retrieveReq.pFuncNames, i); SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC; - mError("func:%s, failed to retrieve since %s", funcName, terrstr()); - goto FUNC_RETRIEVE_OVER; + goto RETRIEVE_FUNC_OVER; } - SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; - memcpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); - pFuncInfo->funcType = pFunc->funcType; - pFuncInfo->scriptType = pFunc->scriptType; - pFuncInfo->outputType = pFunc->outputType; - pFuncInfo->outputLen = htonl(pFunc->outputLen); - pFuncInfo->bufSize = htonl(pFunc->bufSize); - pFuncInfo->signature = htobe64(pFunc->signature); - pFuncInfo->commentSize = htonl(pFunc->commentSize); - pFuncInfo->codeSize = htonl(pFunc->codeSize); - memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize); - memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize); - pOutput += (sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize); + SFuncInfo funcInfo = {0}; + memcpy(funcInfo.name, pFunc->name, TSDB_FUNC_NAME_LEN); + funcInfo.funcType = pFunc->funcType; + funcInfo.scriptType = pFunc->scriptType; + funcInfo.outputType = pFunc->outputType; + funcInfo.outputLen = pFunc->outputLen; + funcInfo.bufSize = pFunc->bufSize; + funcInfo.signature = pFunc->signature; + funcInfo.commentSize = pFunc->commentSize; + funcInfo.codeSize = pFunc->codeSize; + memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); + memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); + taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); mndReleaseFunc(pMnode, pFunc); } - pReq->pCont = pRetrieveRsp; - pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + + tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); + + pReq->pCont = pRsp; + pReq->contLen = contLen; code = 0; -FUNC_RETRIEVE_OVER: - if (code != 0) rpcFreeCont(pRetrieveRsp); +RETRIEVE_FUNC_OVER: + taosArrayDestroy(retrieveReq.pFuncNames); + taosArrayDestroy(retrieveRsp.pFuncInfos); return code; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 57b2e22e77..0f27ebdd71 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -415,7 +415,7 @@ static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { goto CREATE_MNODE_OVER; } - if (mndCheckDropNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto CREATE_MNODE_OVER; } @@ -425,7 +425,6 @@ static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { CREATE_MNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); - return -1; } mndReleaseMnode(pMnode, pObj); @@ -583,7 +582,7 @@ static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { goto DROP_MNODE_OVER; } - if (mndCheckCreateNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto DROP_MNODE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 344f0d9e40..cdb2b79e0f 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -293,7 +293,7 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { goto CREATE_QNODE_OVER; } - if (mndCheckDropNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto CREATE_QNODE_OVER; } @@ -303,7 +303,6 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { CREATE_QNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); - return -1; } mndReleaseQnode(pMnode, pObj); @@ -401,7 +400,7 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { goto DROP_QNODE_OVER; } - if (mndCheckCreateNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto DROP_QNODE_OVER; } @@ -411,7 +410,6 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { DROP_QNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; } mndReleaseQnode(pMnode, pObj); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 0e33daed9f..a239564fe6 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -294,7 +294,7 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { goto CREATE_SNODE_OVER; } - if (mndCheckDropNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto CREATE_SNODE_OVER; } @@ -403,7 +403,7 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { goto DROP_SNODE_OVER; } - if (mndCheckCreateNodeAuth(pUser)) { + if (mndCheckOperateNodeAuth(pUser)) { goto DROP_SNODE_OVER; } @@ -413,7 +413,6 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { DROP_SNODE_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); - return -1; } mndReleaseSnode(pMnode, pObj); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 9d31ec7314..f34f77de0c 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -43,123 +43,147 @@ TEST_F(MndTestFunc, 01_Show_Func) { TEST_F(MndTestFunc, 02_Create_Func) { { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, ""); + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, ""); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN + 1; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN + 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } { - int32_t contLen = sizeof(SCreateFuncReq); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN + 1; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN - 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } { - int32_t contLen = sizeof(SCreateFuncReq) + 24; + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); - pReq->pCont[0] = 0; SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } { - int32_t contLen = sizeof(SCreateFuncReq) + 24; + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + createReq.pCode[0] = 'a'; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); - pReq->pCont[0] = 'a'; SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); } { - int32_t contLen = sizeof(SCreateFuncReq) + 24; + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + createReq.pCode[0] = 'a'; + createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); - pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); - pReq->pCont[0] = 'a'; - pReq->bufSize = htonl(TSDB_FUNC_BUF_SIZE + 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); } for (int32_t i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateFuncReq); - int32_t commentSize = TSDB_FUNC_COMMENT_LEN; - int32_t codeSize = TSDB_FUNC_CODE_LEN; - contLen = (contLen + codeSize + commentSize); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + createReq.codeSize = TSDB_FUNC_CODE_LEN; + createReq.pCode[0] = 'a'; + createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; + createReq.igExists = 0; + if (i == 2) createReq.igExists = 1; + createReq.funcType = 1; + createReq.scriptType = 2; + createReq.outputType = TSDB_DATA_TYPE_SMALLINT; + createReq.outputLen = 12; + createReq.bufSize = 4; + createReq.signature = 5; + for (int32_t i = 0; i < createReq.commentSize - 1; ++i) { + createReq.pComment[i] = 'm'; + } + for (int32_t i = 0; i < createReq.codeSize - 1; ++i) { + createReq.pCode[i] = 'd'; + } - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); - pReq->igExists = 0; - if (i == 2) pReq->igExists = 1; - pReq->funcType = 1; - pReq->scriptType = 2; - pReq->outputType = TSDB_DATA_TYPE_SMALLINT; - pReq->outputLen = htonl(12); - pReq->bufSize = htonl(4); - pReq->signature = htobe64(5); - pReq->commentSize = htonl(commentSize); - pReq->codeSize = htonl(codeSize); - for (int32_t i = 0; i < commentSize - 1; ++i) { - pReq->pCont[i] = 'm'; - } - for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { - pReq->pCont[i] = 'd'; - } + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -187,27 +211,26 @@ TEST_F(MndTestFunc, 02_Create_Func) { TEST_F(MndTestFunc, 03_Retrieve_Func) { { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f1"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(1); - strcpy(pReq->pFuncNames, "f1"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; - pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); EXPECT_STREQ(pFuncInfo->name, "f1"); EXPECT_EQ(pFuncInfo->funcType, 1); @@ -219,9 +242,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { comments[i] = 'm'; } @@ -229,17 +250,20 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { codes[i] = 'd'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); + taosArrayDestroy(retrieveRsp.pFuncInfos); } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 0; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 0; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -247,12 +271,17 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; + retrieveReq.pFuncNames = taosArrayInit(TSDB_FUNC_MAX_RETRIEVE + 1, TSDB_FUNC_NAME_LEN); + for (int32_t i = 0; i < TSDB_FUNC_MAX_RETRIEVE + 1; ++i) { + taosArrayPush(retrieveReq.pFuncNames, "1"); + } - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -260,13 +289,15 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); - strcpy(pReq->pFuncNames, "f2"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -274,28 +305,27 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SCreateFuncReq); - int32_t commentSize = 1024; - int32_t codeSize = 9527; - contLen = (contLen + codeSize + commentSize); + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f2"); + createReq.commentSize = 1024; + createReq.codeSize = 9527; + createReq.igExists = 1; + createReq.funcType = 2; + createReq.scriptType = 3; + createReq.outputType = TSDB_DATA_TYPE_BINARY; + createReq.outputLen = 24; + createReq.bufSize = 6; + createReq.signature = 18; + for (int32_t i = 0; i < createReq.commentSize - 1; ++i) { + createReq.pComment[i] = 'p'; + } + for (int32_t i = 0; i < createReq.codeSize - 1; ++i) { + createReq.pCode[i] = 'q'; + } - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f2"); - pReq->igExists = 1; - pReq->funcType = 2; - pReq->scriptType = 3; - pReq->outputType = TSDB_DATA_TYPE_BINARY; - pReq->outputLen = htonl(24); - pReq->bufSize = htonl(6); - pReq->signature = htobe64(18); - pReq->commentSize = htonl(commentSize); - pReq->codeSize = htonl(codeSize); - for (int32_t i = 0; i < commentSize - 1; ++i) { - pReq->pCont[i] = 'p'; - } - for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { - pReq->pCont[i] = 'q'; - } + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -309,27 +339,26 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 1; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(1); - strcpy(pReq->pFuncNames, "f2"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; - pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); EXPECT_STREQ(pFuncInfo->name, "f2"); EXPECT_EQ(pFuncInfo->funcType, 2); @@ -341,47 +370,43 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, 1024); EXPECT_EQ(pFuncInfo->codeSize, 9527); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char* comments = (char*)calloc(1, 1024); + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < 1024 - 1; ++i) { comments[i] = 'p'; } - char* codes = (char*)calloc(1, 9527); + char codes[TSDB_FUNC_CODE_LEN] = {0}; for (int32_t i = 0; i < 9527 - 1; ++i) { codes[i] = 'q'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); - free(comments); - free(codes); + + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); + taosArrayDestroy(retrieveRsp.pFuncInfos); } { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 2; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 2; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); + taosArrayPush(retrieveReq.pFuncNames, "f1"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); - strcpy(pReq->pFuncNames, "f2"); - strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; - pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 2); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); { - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); - + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); EXPECT_STREQ(pFuncInfo->name, "f2"); EXPECT_EQ(pFuncInfo->funcType, 2); EXPECT_EQ(pFuncInfo->scriptType, 3); @@ -392,29 +417,21 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, 1024); EXPECT_EQ(pFuncInfo->codeSize, 9527); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char* comments = (char*)calloc(1, 1024); + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < 1024 - 1; ++i) { comments[i] = 'p'; } - char* codes = (char*)calloc(1, 9527); + char codes[TSDB_FUNC_CODE_LEN] = {0}; for (int32_t i = 0; i < 9527 - 1; ++i) { codes[i] = 'q'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); - free(comments); - free(codes); + + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); } { - SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos + sizeof(SFuncInfo) + 1024 + 9527); - pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); - pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); - pFuncInfo->signature = htobe64(pFuncInfo->signature); - pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); - pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 1); EXPECT_STREQ(pFuncInfo->name, "f1"); EXPECT_EQ(pFuncInfo->funcType, 1); EXPECT_EQ(pFuncInfo->scriptType, 2); @@ -425,9 +442,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); - char* pComment = pFuncInfo->pCont; - char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { comments[i] = 'm'; } @@ -435,33 +450,38 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { codes[i] = 'd'; } - EXPECT_STREQ(pComment, comments); - EXPECT_STREQ(pCode, codes); + EXPECT_STREQ(comments, pFuncInfo->pComment); + EXPECT_STREQ(codes, pFuncInfo->pCode); } + taosArrayDestroy(retrieveRsp.pFuncInfos); } - { - int32_t contLen = sizeof(SRetrieveFuncReq); - int32_t numOfFuncs = 2; - contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 2; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "f2"); + taosArrayPush(retrieveReq.pFuncNames, "f3"); - SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(numOfFuncs); - strcpy(pReq->pFuncNames, "f2"); - strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f3"); + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); - } + } } TEST_F(MndTestFunc, 04_Drop_Func) { { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, ""); - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, ""); + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -469,10 +489,12 @@ TEST_F(MndTestFunc, 04_Drop_Func) { } { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, "f3"); - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f3"); + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -480,11 +502,13 @@ TEST_F(MndTestFunc, 04_Drop_Func) { } { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, "f3"); + dropReq.igNotExists = 1; - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f3"); - pReq->igNotExists = 1; + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -492,10 +516,13 @@ TEST_F(MndTestFunc, 04_Drop_Func) { } { - int32_t contLen = sizeof(SDropFuncReq); + SDropFuncReq dropReq = {0}; + strcpy(dropReq.name, "f1"); + dropReq.igNotExists = 1; - SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "f1"); + int32_t contLen = tSerializeSDropFuncReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropFuncReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -517,5 +544,5 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); - CheckBinary("f2", TSDB_FUNC_NAME_LEN); + CheckBinary("f2", TSDB_FUNC_NAME_LEN); } From 1755559d94ff9cf987be54f51b66bed3c6bb6510 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Feb 2022 17:00:40 +0800 Subject: [PATCH 12/29] serialize db msg --- include/common/tmsg.h | 22 ++- source/common/src/tmsg.c | 210 +++++++++++++++++++++++++ source/dnode/mnode/impl/inc/mndAuth.h | 8 +- source/dnode/mnode/impl/src/mndAuth.c | 17 +- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndDnode.c | 4 +- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- 10 files changed, 261 insertions(+), 20 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9e99d6a9ba..1bd347fc24 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -570,6 +570,9 @@ typedef struct { int8_t ignoreExist; } SCreateDbReq; +int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); +int32_t tDeserializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t totalBlocks; @@ -582,28 +585,39 @@ typedef struct { int8_t cacheLastRow; } SAlterDbReq; +int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); +int32_t tDeserializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int8_t ignoreNotExists; } SDropDbReq; +int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); +int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; uint64_t uid; } SDropDbRsp; +int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); +int32_t tDeserializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t vgVersion; } SUseDbReq; -typedef struct { - char db[TSDB_DB_FNAME_LEN]; -} SSyncDbReq; +int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); +int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); typedef struct { char db[TSDB_DB_FNAME_LEN]; -} SCompactDbReq; +} SSyncDbReq, SCompactDbReq; + +int32_t tSerializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq); +int32_t tDeserializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c6096ac4b8..20e1da9d07 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1097,3 +1097,213 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp tCoderClear(&decoder); return 0; } + +int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfVgroups) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; + if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; + if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; + if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; + if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; + if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->update) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfVgroups) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; + if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI8(&encoder, pReq->ignoreNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->ignoreNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index ca196f69be..c2c69f000b 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -29,8 +29,12 @@ int32_t mndCheckCreateUserAuth(SUserObj *pOperUser); int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, SAlterUserReq *pAlter); int32_t mndCheckDropUserAuth(SUserObj *pOperUser); -int32_t mndCheckOperateNodeAuth(SUserObj *pOperUser); -int32_t mndCheckOperateFuncAuth(SUserObj *pOperUser); +int32_t mndCheckNodeAuth(SUserObj *pOperUser); +int32_t mndCheckFuncAuth(SUserObj *pOperUser); + +int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); +int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb); +int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 14dc451686..84c8b7476c 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -111,7 +111,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { return -1; } -int32_t mndCheckOperateNodeAuth(SUserObj *pOperUser) { +int32_t mndCheckNodeAuth(SUserObj *pOperUser) { if (pOperUser->superUser) { return 0; } @@ -120,7 +120,7 @@ int32_t mndCheckOperateNodeAuth(SUserObj *pOperUser) { return -1; } -int32_t mndCheckOperateFuncAuth(SUserObj *pOperUser) { +int32_t mndCheckFuncAuth(SUserObj *pOperUser) { if (pOperUser->superUser) { return 0; } @@ -128,3 +128,16 @@ int32_t mndCheckOperateFuncAuth(SUserObj *pOperUser) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } + +int32_t mndCheckCreateDbAuth(SUserObj *pOperUser) { return 0; } + +int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb) { + if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { + return 0; + } + + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} + +int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb) { return 0; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 7bffc482bb..eb9af7b358 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -293,7 +293,7 @@ static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { goto CREATE_BNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto CREATE_BNODE_OVER; } @@ -400,7 +400,7 @@ static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { goto DROP_BNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto DROP_BNODE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3f4213064a..5126ca9db3 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -496,7 +496,7 @@ static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { goto CREATE_DNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto CREATE_DNODE_OVER; } @@ -570,7 +570,7 @@ static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { goto DROP_DNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto DROP_DNODE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index cd5eaca638..3527f103db 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -321,7 +321,7 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { goto CREATE_FUNC_OVER; } - if (mndCheckOperateFuncAuth(pUser)) { + if (mndCheckFuncAuth(pUser)) { goto CREATE_FUNC_OVER; } @@ -376,7 +376,7 @@ static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { goto DROP_FUNC_OVER; } - if (mndCheckOperateFuncAuth(pUser)) { + if (mndCheckFuncAuth(pUser)) { goto DROP_FUNC_OVER; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 0f27ebdd71..eb9ba49dd2 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -415,7 +415,7 @@ static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { goto CREATE_MNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto CREATE_MNODE_OVER; } @@ -582,7 +582,7 @@ static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { goto DROP_MNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto DROP_MNODE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index cdb2b79e0f..db1ea2ab17 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -293,7 +293,7 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { goto CREATE_QNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto CREATE_QNODE_OVER; } @@ -400,7 +400,7 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { goto DROP_QNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto DROP_QNODE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a239564fe6..dd699364c0 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -294,7 +294,7 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { goto CREATE_SNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto CREATE_SNODE_OVER; } @@ -403,7 +403,7 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { goto DROP_SNODE_OVER; } - if (mndCheckOperateNodeAuth(pUser)) { + if (mndCheckNodeAuth(pUser)) { goto DROP_SNODE_OVER; } From d7fc732de3eac60499fabf4e2a38de168c540244 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Feb 2022 17:14:33 +0800 Subject: [PATCH 13/29] serailze db req --- source/dnode/mnode/impl/src/mndDb.c | 105 +++++++++++---------- source/dnode/mnode/impl/test/db/db.cpp | 92 +++++++++--------- source/dnode/mnode/impl/test/stb/stb.cpp | 46 ++++----- source/dnode/mnode/impl/test/user/user.cpp | 46 ++++----- source/libs/catalog/test/catalogTests.cpp | 98 +++++++++---------- source/util/src/terror.c | 2 +- 6 files changed, 196 insertions(+), 193 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5ed3e9e8df..1a01c3852a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -369,7 +370,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pReq; action.contLen = sizeof(SDropVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -451,54 +452,54 @@ CREATE_DB_OVER: } static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCreateDbReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCreateDbReq createReq = {0}; - pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); - pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); - pCreate->totalBlocks = htonl(pCreate->totalBlocks); - pCreate->daysPerFile = htonl(pCreate->daysPerFile); - pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); - pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); - pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRows = htonl(pCreate->minRows); - pCreate->maxRows = htonl(pCreate->maxRows); - pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + if (tDeserializeSCreateDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_DB_OVER; + } - mDebug("db:%s, start to create, vgroups:%d", pCreate->db, pCreate->numOfVgroups); + mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); - SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); + pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { - mndReleaseDb(pMnode, pDb); - if (pCreate->ignoreExist) { - mDebug("db:%s, already exist, ignore exist is set", pCreate->db); - return 0; + if (createReq.ignoreExist) { + mDebug("db:%s, already exist, ignore exist is set", createReq.db); + code = 0; + goto CREATE_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } - SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); - if (pOperUser == NULL) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto CREATE_DB_OVER; } - int32_t code = mndCreateDb(pMnode, pReq, pCreate, pOperUser); - mndReleaseUser(pMnode, pOperUser); - - if (code != 0) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + if (mndCheckCreateDbAuth(pUser) != 0) { + goto CREATE_DB_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + code = mndCreateDb(pMnode, pReq, &createReq, pUser); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to create since %s", createReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { @@ -818,7 +819,7 @@ static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgList, int32_t *vgNum) { int32_t vindex = 0; - SSdb *pSdb = pMnode->pSdb; + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (vindex < pDb->cfg.numOfVgroups) { @@ -833,9 +834,9 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgLis pInfo->hashEnd = htonl(pVgroup->hashEnd); pInfo->epset.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEp * pEp = &pInfo->epset.eps[gid]; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; + SEp *pEp = &pInfo->epset.eps[gid]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); pEp->port = htons(pDnode->port); @@ -895,12 +896,12 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { } int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen) { - SSdb *pSdb = pMnode->pSdb; - int32_t bufSize = num * (sizeof(SUseDbRsp) + TSDB_DEFAULT_VN_PER_DB * sizeof(SVgroupInfo)); - void *buf = malloc(bufSize); - int32_t len = 0; - int32_t contLen = 0; - int32_t bufOffset = 0; + SSdb *pSdb = pMnode->pSdb; + int32_t bufSize = num * (sizeof(SUseDbRsp) + TSDB_DEFAULT_VN_PER_DB * sizeof(SVgroupInfo)); + void *buf = malloc(bufSize); + int32_t len = 0; + int32_t contLen = 0; + int32_t bufOffset = 0; SUseDbRsp *pRsp = NULL; for (int32_t i = 0; i < num; ++i) { @@ -909,11 +910,11 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * db->vgVersion = ntohl(db->vgVersion); len = 0; - + SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); if (pDb == NULL) { mInfo("db %s not exist", db->dbFName); - + len = sizeof(SUseDbRsp); } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { len = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); @@ -921,16 +922,16 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * if (0 == len) { mndReleaseDb(pMnode, pDb); - + continue; } - + contLen += len; - + if (contLen > bufSize) { buf = realloc(buf, contLen); } - + pRsp = (SUseDbRsp *)((char *)buf + bufOffset); memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); if (pDb) { @@ -949,7 +950,7 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * } bufOffset += len; - + mndReleaseDb(pMnode, pDb); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 2d1574467e..4c09caed9c 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -53,29 +53,31 @@ TEST_F(MndTestDb, 01_ShowDb) { TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d1"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -217,29 +219,31 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index c5ab44a014..6d0005fd15 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -38,29 +38,31 @@ class MndTestStb : public ::testing::Test { Testbase MndTestStb::test; void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, dbname); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); *pContLen = contLen; return pReq; diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 8f9edfab09..d8ce599be1 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -319,29 +319,31 @@ TEST_F(MndTestUser, 03_Alter_User) { } { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index d0f98e3c2a..4eeae0a244 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -39,7 +39,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist); extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); -extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); +extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type); void ctgTestSetPrepareTableMeta(); void ctgTestSetPrepareCTableMeta(); @@ -52,14 +52,14 @@ bool ctgTestDeadLoop = false; int32_t ctgTestPrintNum = 200000; int32_t ctgTestMTRunSec = 5; -int32_t ctgTestCurrentVgVersion = 0; -int32_t ctgTestVgVersion = 1; -int32_t ctgTestVgNum = 10; -int32_t ctgTestColNum = 2; -int32_t ctgTestTagNum = 1; -int32_t ctgTestSVersion = 1; -int32_t ctgTestTVersion = 1; -int32_t ctgTestSuid = 2; +int32_t ctgTestCurrentVgVersion = 0; +int32_t ctgTestVgVersion = 1; +int32_t ctgTestVgNum = 10; +int32_t ctgTestColNum = 2; +int32_t ctgTestTagNum = 1; +int32_t ctgTestSVersion = 1; +int32_t ctgTestTVersion = 1; +int32_t ctgTestSuid = 2; uint64_t ctgTestDbId = 33; uint64_t ctgTestClusterId = 0x1; @@ -69,31 +69,35 @@ char *ctgTestCTablename = "ctable1"; char *ctgTestSTablename = "stable1"; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { - SCreateDbReq *pReq = (SCreateDbReq *)rpcMallocCont(sizeof(SCreateDbReq)); - strcpy(pReq->db, "1.db1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.db1"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbReq); + rpcMsg.contLen = contLen; rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; @@ -210,7 +214,6 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { *pdbVgroup = dbVgroup; } - void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { strcpy(rspMsg->dbFName, ctgTestDbname); sprintf(rspMsg->tbName, "%s", ctgTestSTablename); @@ -248,7 +251,6 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { return; } - void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SUseDbRsp *rspMsg = NULL; // todo @@ -372,8 +374,8 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; strcpy(rspMsg->dbFName, ctgTestDbname); - strcpy(rspMsg->tbName, ctgTestSTablename); - strcpy(rspMsg->stbName, ctgTestSTablename); + strcpy(rspMsg->tbName, ctgTestSTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -635,7 +637,7 @@ void *ctgTestGetDbVgroupThread(void *param) { void *ctgTestSetSameDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgroupInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { @@ -656,11 +658,10 @@ void *ctgTestSetSameDbVgroupThread(void *param) { return NULL; } - void *ctgTestSetDiffDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgroupInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { @@ -681,7 +682,6 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { return NULL; } - void *ctgTestGetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; @@ -741,7 +741,6 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } - TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -914,7 +913,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -1016,7 +1015,7 @@ TEST(tableMeta, superTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - + free(stb); stb = NULL; } else { @@ -1078,7 +1077,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); - + catalogDestroy(); } @@ -1146,12 +1145,10 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); tfree(tableMeta); - + catalogDestroy(); } - - TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1258,7 +1255,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgroupInfo *dbVgroup = NULL; SArray *vgList = NULL; ctgTestInitLogFile(); @@ -1418,8 +1415,6 @@ TEST(multiThread, getSetRmDiffDbVgroup) { catalogDestroy(); } - - TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1470,8 +1465,6 @@ TEST(multiThread, ctableMeta) { catalogDestroy(); } - - TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1530,7 +1523,8 @@ TEST(rentTest, allRent) { printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, + stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4cb1e392b9..70608e117e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -77,7 +77,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") From 7dba7215f46876897829731ded0e18e7039c45b5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 12 Feb 2022 18:14:28 +0800 Subject: [PATCH 14/29] add demo --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/trans.c | 3 + source/libs/transport/src/transCli.c | 30 ++-- source/libs/transport/src/transSrv.c | 6 +- source/libs/transport/test/CMakeLists.txt | 18 ++ source/libs/transport/test/pushServer.c | 199 ++++++++++++++++++++++ 6 files changed, 241 insertions(+), 16 deletions(-) create mode 100644 source/libs/transport/test/pushServer.c diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index cc65f04a39..bec0375dbe 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -173,6 +173,7 @@ typedef struct { typedef struct { uint8_t user[TSDB_UNI_LEN]; + uint8_t secret[TSDB_PASSWORD_LEN]; } STransUserMsg; #pragma pack(pop) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index b03adafaff..707f23113b 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -39,6 +39,9 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->user) { memcpy(pRpc->user, pInit->user, strlen(pInit->user)); } + if (pInit->secret) { + memcpy(pRpc->secret, pInit->secret, strlen(pInit->secret)); + } return pRpc; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9d13ec8225..2a4a1891ed 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -135,9 +135,10 @@ static void clientHandleResp(SCliConn* conn) { if (conn->push != NULL && conn->notifyCount != 0) { (*conn->push->callback)(conn->push->arg, &rpcMsg); + conn->push = NULL; } else { if (pCtx->pSem == NULL) { - tTrace("client conn(sync) %p handle resp", conn); + tTrace("client conn%p handle resp", conn); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); } else { tTrace("client conn(sync) %p handle resp", conn); @@ -146,6 +147,7 @@ static void clientHandleResp(SCliConn* conn) { } } conn->notifyCount += 1; + conn->secured = pHead->secured; // buf's mem alread translated to rpcMsg.pCont transClearBuffer(&conn->readBuf); @@ -156,10 +158,10 @@ static void clientHandleResp(SCliConn* conn) { // user owns conn->persist = 1 if (conn->push == NULL) { addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); - } - destroyCmsg(pMsg); - conn->data = NULL; + destroyCmsg(conn->data); + conn->data = NULL; + } // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); @@ -182,21 +184,23 @@ static void clientHandleExcept(SCliConn* pConn) { if (pConn->push != NULL && pConn->notifyCount != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); + pConn->push = NULL; } else { if (pCtx->pSem == NULL) { (pCtx->pTransInst->cfp)(pCtx->pTransInst->parent, &rpcMsg, NULL); } else { memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg)); - // SRpcMsg rpcMsg tsem_post(pCtx->pSem); } if (pConn->push != NULL) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); } + pConn->push = NULL; + } + if (pConn->push == NULL) { + destroyCmsg(pConn->data); + pConn->data = NULL; } - - destroyCmsg(pMsg); - pConn->data = NULL; // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); pConn->notifyCount += 1; @@ -383,6 +387,7 @@ static void clientWriteCb(uv_write_t* req, int status) { static void clientWrite(SCliConn* pConn) { SCliMsg* pCliMsg = pConn->data; STransConnCtx* pCtx = pCliMsg->ctx; + SRpcInfo* pTransInst = pCtx->pTransInst; SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); @@ -394,7 +399,8 @@ static void clientWrite(SCliConn* pConn) { memcpy(buf, (char*)pHead, msgLen); STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); - memcpy(uMsg->user, pCtx->pTransInst->user, tListLen(uMsg->user)); + memcpy(uMsg->user, pTransInst->user, tListLen(uMsg->user)); + memcpy(uMsg->secret, pTransInst->secret, tListLen(uMsg->secret)); // to avoid mem leak destroyUserdata(pMsg); @@ -402,8 +408,6 @@ static void clientWrite(SCliConn* pConn) { pMsg->pCont = (char*)buf + sizeof(STransMsgHead); pMsg->contLen = msgLen + sizeof(STransUserMsg) - sizeof(STransMsgHead); - pConn->secured = 1; // del later - pHead = (STransMsgHead*)buf; pHead->secured = 1; msgLen += sizeof(STransUserMsg); @@ -450,10 +454,6 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("client msg tran time cost: %" PRIu64 "", el); et = taosGetTimestampUs(); - // if (pMsg->msg.handle != NULL) { - // // handle - //} - STransConnCtx* pCtx = pMsg->ctx; SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index cc9933f919..5f4daef344 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -234,6 +234,7 @@ static void uvHandleReq(SSrvConn* pConn) { if (pHead->secured == 1) { STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg)); memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); + memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); } pConn->inType = pHead->msgType; @@ -335,13 +336,16 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); - SRpcMsg* pMsg = &smsg->msg; + SSrvConn* pConn = smsg->pConn; + SRpcMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + + pHead->secured = pMsg->code == 0 ? 1 : 0; // pHead->msgType = smsg->pConn->inType + 1; pHead->code = htonl(pMsg->code); // add more info diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index 26b45ed09b..b4f50219ff 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(server "") add_executable(transUT "") add_executable(syncClient "") add_executable(pushClient "") +add_executable(pushServer "") target_sources(transUT PRIVATE @@ -30,6 +31,10 @@ target_sources(pushClient PRIVATE "pushClient.c" ) +target_sources(pushServer + PRIVATE + "pushServer.c" +) target_include_directories(transportTest PUBLIC @@ -110,3 +115,16 @@ target_link_libraries (pushClient transport ) +target_include_directories(pushServer + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries (pushServer + os + util + common + gtest_main + transport +) + diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c new file mode 100644 index 0000000000..f9115d3d4f --- /dev/null +++ b/source/libs/transport/test/pushServer.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +//#define _DEFAULT_SOURCE +#include "os.h" +#include "rpcLog.h" +#include "tglobal.h" +#include "tqueue.h" +#include "trpc.h" + +int msgSize = 128; +int commit = 0; +int dataFd = -1; +STaosQueue *qhandle = NULL; +STaosQset * qset = NULL; + +void processShellMsg() { + static int num = 0; + STaosQall *qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + void * pvnode; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode, NULL); + tDebug("%d shell msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + + if (dataFd >= 0) { + if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + tInfo("failed to write data file, reason:%s", strerror(errno)); + } + } + } + + if (commit >= 2) { + num += numOfMsgs; + // if (taosFsync(dataFd) < 0) { + // tInfo("failed to flush data to file, reason:%s", strerror(errno)); + //} + + if (num % 10000 == 0) { + tInfo("%d request have been written into disk", num); + } + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + void *handle = pRpcMsg->handle; + taosFreeQitem(pRpcMsg); + + { + sleep(1); + SRpcMsg nRpcMsg = {0}; + nRpcMsg.pCont = rpcMallocCont(msgSize); + nRpcMsg.contLen = msgSize; + nRpcMsg.handle = handle; + nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; + rpcSendResponse(&nRpcMsg); + } + } + } + + taosFreeQall(qall); +} + +int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + + if (strcmp(meterId, "michael") == 0) { + *spi = 1; + *encrypt = 0; + strcpy(secret, "mypassword"); + strcpy(ckey, "key"); + } else if (strcmp(meterId, "jeff") == 0) { + *spi = 0; + *encrypt = 0; + } else { + ret = -1; // user not there + } + + return ret; +} + +void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(qhandle, pTemp); +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + char dataName[20] = "server.data"; + + taosBlockSIGPIPE(); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 7000; + rpcInit.label = "SER"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = 2 * 1500; + rpcInit.afp = retrieveAuthInfo; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { + rpcInit.localPort = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { + rpcInit.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { + msgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { + rpcInit.sessions = atoi(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { + tsCompressMsgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { + commit = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { + rpcDebugFlag = atoi(argv[++i]); + dDebugFlag = rpcDebugFlag; + uDebugFlag = rpcDebugFlag; + } else { + printf("\nusage: %s [options] \n", argv[0]); + printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); + printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); + printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); + printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); + printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); + printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); + printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); + printf(" [-h help]: print out this help\n\n"); + exit(0); + } + } + + tsAsyncLog = 0; + rpcInit.connType = TAOS_CONN_SERVER; + taosInitLog("server.log", 100000, 10); + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + tError("failed to start RPC server"); + return -1; + } + // sleep(5); + + tInfo("RPC server is running, ctrl-c to exit"); + + if (commit) { + dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + } + qhandle = taosOpenQueue(); + qset = taosOpenQset(); + taosAddIntoQset(qset, qhandle, NULL); + + processShellMsg(); + + if (dataFd >= 0) { + close(dataFd); + remove(dataName); + } + + return 0; +} From 48495592c1b988e0ee46782a3193b4b8c26f63fd Mon Sep 17 00:00:00 2001 From: dapan Date: Sat, 12 Feb 2022 20:03:42 +0800 Subject: [PATCH 15/29] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 20 +- source/libs/catalog/src/catalog.c | 425 +++++++++++++++------- source/libs/catalog/test/catalogTests.cpp | 306 ++++++++++++---- 3 files changed, 553 insertions(+), 198 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 6bfc8d6ca3..f427910758 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -105,7 +105,8 @@ typedef struct SCtgApiStat { } SCtgApiStat; typedef struct SCtgRuntimeStat { - + uint64_t qNum; + uint64_t qDoneNum; } SCtgRuntimeStat; typedef struct SCtgCacheStat { @@ -161,6 +162,7 @@ typedef struct SCatalogMgmt { SCtgQNode *head; SCtgQNode *tail; tsem_t sem; + uint64_t qRemainNum; pthread_t updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; @@ -170,6 +172,18 @@ typedef struct SCatalogMgmt { typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); typedef int32_t (*ctgActFunc)(SCtgMetaAction *); +typedef struct SCtgAction { + int32_t actId; + char name[32]; + ctgActFunc func; +} SCtgAction; + +#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.qRemainNum, 1) +#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.qRemainNum, 1) + +#define CTG_STAT_ADD(n) qError("done:%" PRId64, atomic_add_fetch_64(&(n), 1)) +#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) + #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) @@ -236,8 +250,8 @@ typedef int32_t (*ctgActFunc)(SCtgMetaAction *); #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define CTG_API_ENTER() do { CTG_API_DEBUG("enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) -#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_API_DEBUG("leave %s", __FUNCTION__); CTG_RET(__code); } while (0) +#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) +#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8(&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 5195a0adf8..732d3ab111 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,11 +24,69 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); -SCatalogMgmt ctgMgmt = {0}; - +SCatalogMgmt gCtgMgmt = {0}; SCtgDebug gCTGDebug = {0}; +SCtgAction gCtgAction[CTG_ACT_MAX] = {{ + CTG_ACT_UPDATE_VG, + "update vgInfo", + ctgActUpdateVg + }, + { + CTG_ACT_UPDATE_TBL, + "update tbMeta", + ctgActUpdateTbl + }, + { + CTG_ACT_REMOVE_DB, + "remove DB", + ctgActRemoveDB + }, + { + CTG_ACT_REMOVE_STB, + "remove stbMeta", + ctgActRemoveStb + }, + { + CTG_ACT_REMOVE_TBL, + "remove tbMeta", + ctgActRemoveTbl + } +}; -ctgActFunc ctgActFuncs[] = {ctgActUpdateVg, ctgActUpdateTbl, ctgActRemoveDB, ctgActRemoveStb, ctgActRemoveTbl}; +int32_t ctgDbgEnableDebug(char *option) { + if (0 == strcasecmp(option, "lock")) { + gCTGDebug.lockDebug = true; + qDebug("lock debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "cache")) { + gCTGDebug.cacheDebug = true; + qDebug("cache debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "api")) { + gCTGDebug.apiDebug = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + qError("invalid debug option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgDbgGetStatNum(char *option, void *res) { + if (0 == strcasecmp(option, "runtime.qDoneNum")) { + *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); + return TSDB_CODE_SUCCESS; + } + + qError("invalid stat option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; @@ -128,6 +186,45 @@ void ctgDbgShowClusterCache(SCatalog* pCtg) { ctgDbgShowDBCache(pCtg->dbCache); } + + +void ctgPopAction(SCtgMetaAction **action) { + SCtgQNode *orig = gCtgMgmt.head; + + SCtgQNode *node = gCtgMgmt.head->next; + gCtgMgmt.head = gCtgMgmt.head->next; + + CTG_QUEUE_SUB(); + + tfree(orig); + + *action = &node->action; +} + + +int32_t ctgPushAction(SCtgMetaAction *action) { + SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); + if (NULL == node) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + node->action = *action; + + CTG_LOCK(CTG_WRITE, &gCtgMgmt.qlock); + gCtgMgmt.tail->next = node; + gCtgMgmt.tail = node; + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.qlock); + + CTG_QUEUE_ADD(); + //CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + + tsem_post(&gCtgMgmt.sem); + + return TSDB_CODE_SUCCESS; +} + + void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { return; @@ -210,16 +307,29 @@ void ctgFreeHandle(SCatalog* pCtg) { } -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { +int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); + if (dbCache->deleted) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + if (NULL == dbCache->vgInfo) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + *inCache = false; ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); return TSDB_CODE_SUCCESS; } + *inCache = true; + return TSDB_CODE_SUCCESS; } @@ -302,12 +412,11 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac return TSDB_CODE_SUCCESS; } - ctgAcquireVgInfo(pCtg, dbCache); - if (NULL == dbCache->vgInfo) { + ctgAcquireVgInfo(pCtg, dbCache, inCache); + if (!(*inCache)) { ctgReleaseDBCache(pCtg, dbCache); *pCache = NULL; - *inCache = false; return TSDB_CODE_SUCCESS; } @@ -325,7 +434,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE char *msg = NULL; int32_t msgLen = 0; - ctgDebug("try to get db vgroup from mnode, db:%s", input->db); + ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen); if (code) { @@ -358,6 +467,8 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE CTG_ERR_RET(code); } + ctgDebug("Got db vgInfo from mnode, dbFName:%s", input->db); + return TSDB_CODE_SUCCESS; } @@ -381,7 +492,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == tbMeta) { - taosHashRelease(pCtg->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); *exist = 0; ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -390,7 +501,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, *exist = 1; - taosHashRelease(pCtg->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -414,7 +525,6 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { *exist = 0; - ctgWarn("no db cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -486,12 +596,12 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ return TSDB_CODE_SUCCESS; } - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbName); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); - SCtgDBCache *dbCache = taosHashAcquire(pCtg->dbCache, dbName, strlen(dbName)); + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - ctgInfo("db not in cache, dbFName:%s", dbName); return TSDB_CODE_SUCCESS; } @@ -500,8 +610,8 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ if (NULL == pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); - taosHashRelease(pCtg->dbCache, dbCache); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + ctgReleaseDBCache(pCtg, dbCache); return TSDB_CODE_SUCCESS; } @@ -512,9 +622,9 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - taosHashRelease(pCtg->dbCache, dbCache); + ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } @@ -940,15 +1050,15 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache.metaCache) { - ctgError("taosHashInit %d metaCache failed", ctgMgmt.cfg.maxTblCacheNum); + ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache.stbCache) { - ctgError("taosHashInit %d stbCache failed", ctgMgmt.cfg.maxTblCacheNum); + ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1002,6 +1112,10 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { + uint64_t dbId = dbCache->dbId; + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + atomic_store_8(&dbCache->deleted, 1); ctgRemoveStbRent(pCtg, &dbCache->tbCache); @@ -1018,6 +1132,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; } @@ -1160,6 +1276,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + if (!isStb) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; @@ -1183,7 +1301,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgDebug("meta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; strcpy(metaRent.dbFName, dbFName); @@ -1193,6 +1311,45 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui return TSDB_CODE_SUCCESS; } +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { + *dst = malloc(sizeof(SDBVgInfo)); + if (NULL == *dst) { + qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*dst, src, sizeof(SDBVgInfo)); + + size_t hashSize = taosHashGetSize(src->vgHash); + (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == (*dst)->vgHash) { + qError("taosHashInit %d failed", (int32_t)hashSize); + tfree(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + int32_t *vgId = NULL; + void *pIter = taosHashIterate(src->vgHash, NULL); + while (pIter) { + taosHashGetKey(pIter, (void **)&vgId, NULL); + + if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { + qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); + taosHashCancelIterate(src->vgHash, pIter); + taosHashCleanup((*dst)->vgHash); + tfree(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(src->vgHash, pIter); + } + + + return TSDB_CODE_SUCCESS; +} + + + int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; int32_t code = 0; @@ -1209,23 +1366,39 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - while (true) { - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); - code = ctgUpdateDBVgInfo(pCtg, dbFName, DbOut.dbId, &DbOut.dbVgroup); - if (code && DbOut.dbVgroup) { - *pInfo = DbOut.dbVgroup; - return TSDB_CODE_SUCCESS; - } - - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); - - if (inCache) { - return TSDB_CODE_SUCCESS; - } + CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); + + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; + SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + ctgFreeVgInfo(DbOut.dbVgroup); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = DbOut.dbId; + msg->dbInfo = DbOut.dbVgroup; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + return TSDB_CODE_SUCCESS; + +_return: + + tfree(*pInfo); + tfree(msg); + + *pInfo = DbOut.dbVgroup; + + CTG_RET(code); } @@ -1253,36 +1426,6 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } -void ctgPopAction(SCtgMetaAction **action) { - SCtgQNode *orig = ctgMgmt.head; - - SCtgQNode *node = ctgMgmt.head->next; - ctgMgmt.head = ctgMgmt.head->next; - - tfree(orig); - - *action = &node->action; -} - - -int32_t ctgPushAction(SCtgMetaAction *action) { - SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); - if (NULL == node) { - qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); - CTG_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - node->action = *action; - - CTG_LOCK(CTG_WRITE, &ctgMgmt.qlock); - ctgMgmt.tail->next = node; - ctgMgmt.tail = node; - CTG_UNLOCK(CTG_WRITE, &ctgMgmt.qlock); - - tsem_post(&ctgMgmt.sem); - - return TSDB_CODE_SUCCESS; -} int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { @@ -1295,6 +1438,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + SCtgUpdateTblMsg *msg = NULL; STableMetaOutput moutput = {0}; STableMetaOutput *output = malloc(sizeof(STableMetaOutput)); if (NULL == output) { @@ -1303,7 +1447,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm } if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); @@ -1312,13 +1456,13 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { - ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { - ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); tfree(output->tbMeta); @@ -1345,16 +1489,22 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm } if (CTG_IS_META_NULL(output->metaType)) { - ctgError("no tablemeta got, tbNmae:%s", tNameGetTableName(pTableName)); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } + if (CTG_IS_META_TABLE(output->metaType)) { + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); + } else { + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + } + if (pOutput) { CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; - SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + msg = malloc(sizeof(SCtgUpdateTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -1367,11 +1517,14 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm CTG_ERR_JRET(ctgPushAction(&action)); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + return TSDB_CODE_SUCCESS; _return: tfree(output->tbMeta); + tfree(output); tfree(msg); CTG_RET(code); @@ -1440,6 +1593,10 @@ _return: tfree(output); + if (*pTableMeta) { + ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); + } + CTG_RET(code); } @@ -1465,15 +1622,13 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action) { SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(msg->pCtg, msg->dbFName, &dbCache); + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { - ctgInfo("db not exist in cache, may be removed, dbFName:%s", msg->dbFName); goto _return; } if (dbCache->dbId != msg->dbId) { ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); - ctgReleaseDBCache(msg->pCtg, dbCache); goto _return; } @@ -1496,7 +1651,7 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { @@ -1522,10 +1677,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { _return: - if (dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); + if (output) { + tfree(output->tbMeta); + tfree(output); } - + tfree(msg); CTG_RET(code); @@ -1591,22 +1747,26 @@ void* ctgUpdateThreadFunc(void* param) { qInfo("catalog update thread started"); - CTG_LOCK(CTG_READ, &ctgMgmt.lock); + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); while (true) { - tsem_wait(&ctgMgmt.sem); + tsem_wait(&gCtgMgmt.sem); - if (atomic_load_8(&ctgMgmt.exit)) { + if (atomic_load_8(&gCtgMgmt.exit)) { break; } SCtgMetaAction *action = NULL; ctgPopAction(&action); - (*ctgActFuncs[action->act])(action); + qDebug("process %s action", gCtgAction[action->act].name); + + (*gCtgAction[action->act].func)(action); + + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); } - CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); qInfo("catalog update thread stopped"); @@ -1619,7 +1779,7 @@ int32_t ctgStartUpdateThread() { pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&ctgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + if (pthread_create(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } @@ -1630,56 +1790,56 @@ int32_t ctgStartUpdateThread() { int32_t catalogInit(SCatalogCfg *cfg) { - if (ctgMgmt.pCluster) { + if (gCtgMgmt.pCluster) { qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8(&ctgMgmt.exit, false); + atomic_store_8(&gCtgMgmt.exit, false); if (cfg) { - memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); + memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); - if (ctgMgmt.cfg.maxDBCacheNum == 0) { - ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + if (gCtgMgmt.cfg.maxDBCacheNum == 0) { + gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; } - if (ctgMgmt.cfg.maxTblCacheNum == 0) { - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; + if (gCtgMgmt.cfg.maxTblCacheNum == 0) { + gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; } - if (ctgMgmt.cfg.dbRentSec == 0) { - ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; + if (gCtgMgmt.cfg.dbRentSec == 0) { + gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; } - if (ctgMgmt.cfg.stbRentSec == 0) { - ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; + if (gCtgMgmt.cfg.stbRentSec == 0) { + gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } } else { - ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; - ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; - ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; - ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; + gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; + gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; + gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - if (NULL == ctgMgmt.pCluster) { + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } CTG_ERR_RET(ctgStartUpdateThread()); - tsem_init(&ctgMgmt.sem, 0, 0); + tsem_init(&gCtgMgmt.sem, 0, 0); - ctgMgmt.head = calloc(1, sizeof(SCtgQNode)); - if (NULL == ctgMgmt.head) { + gCtgMgmt.head = calloc(1, sizeof(SCtgQNode)); + if (NULL == gCtgMgmt.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - ctgMgmt.tail = ctgMgmt.head; + gCtgMgmt.tail = gCtgMgmt.head; - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } @@ -1689,7 +1849,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == ctgMgmt.pCluster) { + if (NULL == gCtgMgmt.pCluster) { qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -1698,7 +1858,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { SCatalog *clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); + SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; @@ -1714,22 +1874,22 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - clusterCtg->dbCache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == clusterCtg->dbCache) { qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - SHashObj *metaCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == metaCache) { - qError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); + code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { ctgFreeHandle(clusterCtg); @@ -1761,7 +1921,7 @@ void catalogFreeHandle(SCatalog* pCtg) { return; } - if (taosHashRemove(ctgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { + if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); return; } @@ -1793,8 +1953,9 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - ctgAcquireVgInfo(pCtg, dbCache); - if (NULL == dbCache->vgInfo) { + bool inCache = false; + ctgAcquireVgInfo(pCtg, dbCache, &inCache); + if (!inCache) { ctgReleaseDBCache(pCtg, dbCache); *version = CTG_DEFAULT_INVALID_VERSION; @@ -1877,6 +2038,8 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId CTG_ERR_JRET(ctgPushAction(&action)); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_API_LEAVE(code); _return: @@ -1920,6 +2083,8 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_ERR_JRET(ctgPushAction(&action)); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: @@ -1959,6 +2124,8 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_ERR_JRET(ctgPushAction(&action)); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: @@ -2019,6 +2186,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_ERR_JRET(ctgPushAction(&action)); + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_API_LEAVE(code); _return: @@ -2059,6 +2228,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm SVgroupInfo vgroupInfo = {0}; SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; *pVgList = NULL; @@ -2068,7 +2238,6 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm tNameGetFullDbName(pTableName, db); SHashObj *vgHash = NULL; - SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, false, &dbCache, &vgInfo)); if (dbCache) { @@ -2254,16 +2423,18 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) void catalogDestroy(void) { qInfo("start to destroy catalog"); - if (NULL == ctgMgmt.pCluster || atomic_load_8(&ctgMgmt.exit)) { + if (NULL == gCtgMgmt.pCluster || atomic_load_8(&gCtgMgmt.exit)) { return; } - atomic_store_8(&ctgMgmt.exit, true); + atomic_store_8(&gCtgMgmt.exit, true); - CTG_LOCK(CTG_WRITE, &ctgMgmt.lock); + tsem_post(&gCtgMgmt.sem); + + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); SCatalog *pCtg = NULL; - void *pIter = taosHashIterate(ctgMgmt.pCluster, NULL); + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { pCtg = *(SCatalog **)pIter; @@ -2271,13 +2442,13 @@ void catalogDestroy(void) { catalogFreeHandle(pCtg); } - pIter = taosHashIterate(ctgMgmt.pCluster, pIter); + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - taosHashCleanup(ctgMgmt.pCluster); - ctgMgmt.pCluster = NULL; + taosHashCleanup(gCtgMgmt.pCluster); + gCtgMgmt.pCluster = NULL; - CTG_UNLOCK(CTG_WRITE, &ctgMgmt.lock); + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock); qInfo("catalog destroyed"); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index fc97040ffb..3db85b11d0 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -41,11 +41,23 @@ extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SNa int32_t *exist); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); +extern "C" int32_t ctgDbgEnableDebug(char *option); +extern "C" int32_t ctgDbgGetStatNum(char *option, void *res); -void ctgTestSetPrepareTableMeta(); -void ctgTestSetPrepareCTableMeta(); -void ctgTestSetPrepareSTableMeta(); -void ctgTestSetPrepareMultiSTableMeta(); +void ctgTestSetRspTableMeta(); +void ctgTestSetRspCTableMeta(); +void ctgTestSetRspSTableMeta(); +void ctgTestSetRspMultiSTableMeta(); + +extern "C" SCatalogMgmt gCtgMgmt; + +enum { + CTGT_RSP_VGINFO = 1, + CTGT_RSP_TBMETA, + CTGT_RSP_CTBMETA, + CTGT_RSP_STBMETA, + CTGT_RSP_MSTBMETA, +}; bool ctgTestStop = false; bool ctgTestEnableSleep = false; @@ -69,6 +81,9 @@ char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; char *ctgTestSTablename = "stable1"; +int32_t ctgTestRspFunc[10] = {0}; +int32_t ctgTestRspIdx = 0; + void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SCreateDbReq *pReq = (SCreateDbReq *)rpcMallocCont(sizeof(SCreateDbReq)); strcpy(pReq->db, "1.db1"); @@ -111,6 +126,8 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + ctgDbgEnableDebug("api"); + char temp[128] = {0}; sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { @@ -250,7 +267,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { } -void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SUseDbRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -286,7 +303,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM return; } -void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -322,7 +339,7 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM return; } -void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -365,7 +382,7 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc return; } -void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo pRsp->code = 0; @@ -408,7 +425,7 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc return; } -void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp *rspMsg = NULL; // todo static int32_t idx = 1; @@ -454,151 +471,193 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, return; } -void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareTableMeta(); +void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + switch (ctgTestRspFunc[ctgTestRspIdx]) { + case CTGT_RSP_VGINFO: + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_TBMETA: + ctgTestRspTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_CTBMETA: + ctgTestRspCTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_STBMETA: + ctgTestRspSTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + case CTGT_RSP_MSTBMETA: + ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp); + break; + default: + break; + } + + ctgTestRspIdx++; return; } -void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareCTableMeta(); +void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetRspTableMeta(); return; } -void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); +void ctgTestRspDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareSTableMeta(); + ctgTestSetRspCTableMeta(); return; } -void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); +void ctgTestRspDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); - ctgTestSetPrepareMultiSTableMeta(); + ctgTestSetRspSTableMeta(); return; } -void ctgTestSetPrepareDbVgroups() { +void ctgTestRspDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetRspMultiSTableMeta(); + + return; +} + +void ctgTestSetRspDbVgroups() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroups); + stub.set(rpcSendRecv, ctgTestRspDbVgroups); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroups); + stub.set(f.second, ctgTestRspDbVgroups); } } } -void ctgTestSetPrepareTableMeta() { +void ctgTestSetRspTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareTableMeta); + stub.set(rpcSendRecv, ctgTestRspTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareTableMeta); + stub.set(f.second, ctgTestRspTableMeta); } } } -void ctgTestSetPrepareCTableMeta() { +void ctgTestSetRspCTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareCTableMeta); + stub.set(rpcSendRecv, ctgTestRspCTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareCTableMeta); + stub.set(f.second, ctgTestRspCTableMeta); } } } -void ctgTestSetPrepareSTableMeta() { +void ctgTestSetRspSTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareSTableMeta); + stub.set(rpcSendRecv, ctgTestRspSTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareSTableMeta); + stub.set(f.second, ctgTestRspSTableMeta); } } } -void ctgTestSetPrepareMultiSTableMeta() { +void ctgTestSetRspMultiSTableMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta); + stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareMultiSTableMeta); + stub.set(f.second, ctgTestRspMultiSTableMeta); } } } -void ctgTestSetPrepareDbVgroupsAndNormalMeta() { +void ctgTestSetRspByIdx() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta); + stub.set(rpcSendRecv, ctgTestRspByIdx); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta); + stub.set(f.second, ctgTestRspByIdx); } } } -void ctgTestSetPrepareDbVgroupsAndChildMeta() { + +void ctgTestSetRspDbVgroupsAndNormalMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta); } } } -void ctgTestSetPrepareDbVgroupsAndSuperMeta() { +void ctgTestSetRspDbVgroupsAndChildMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta); } } } -void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() { +void ctgTestSetRspDbVgroupsAndSuperMeta() { static Stub stub; - stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta); + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto &f : result) { - stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta); + stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta); + } + } +} + +void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto &f : result) { + stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta); } } } @@ -719,17 +778,19 @@ void *ctgTestSetCtableMetaThread(void *param) { int32_t code = 0; SDBVgInfo dbVgroup = {0}; int32_t n = 0; - STableMetaOutput output = {0}; + STableMetaOutput *output = NULL; - ctgTestBuildCTableMetaOutput(&output); SCtgMetaAction action = {0}; action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { + output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput)); + ctgTestBuildCTableMetaOutput(output); + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); msg->pCtg = pCtg; - msg->output = &output; + msg->output = output; action.data = msg; code = ctgActUpdateTbl(&action); @@ -745,8 +806,6 @@ void *ctgTestSetCtableMetaThread(void *param) { } } - tfree(output.tbMeta); - return NULL; } @@ -758,7 +817,7 @@ TEST(tableMeta, normalTable) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -779,7 +838,11 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epset.numOfEps, 3); - ctgTestSetPrepareTableMeta(); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + usleep(100); + } + + ctgTestSetRspTableMeta(); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -793,6 +856,11 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(100); + } + + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -841,6 +909,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(allStbNum, 0); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, childTableCase) { @@ -850,7 +919,7 @@ TEST(tableMeta, childTableCase) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestSetRspDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -877,6 +946,10 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(100); + } + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -939,6 +1012,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(allStbNum, 1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, superTableCase) { @@ -946,7 +1020,7 @@ TEST(tableMeta, superTableCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -975,7 +1049,11 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - ctgTestSetPrepareCTableMeta(); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(100); + } + + ctgTestSetRspCTableMeta(); tableMeta = NULL; @@ -992,6 +1070,10 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(100); + } + tableMeta = NULL; code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); ASSERT_EQ(code, 0); @@ -1041,6 +1123,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(allStbNum, 1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, rmStbMeta) { @@ -1050,7 +1133,7 @@ TEST(tableMeta, rmStbMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -1079,9 +1162,17 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(100); + } + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); ASSERT_EQ(code, 0); + while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) || ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)) { + usleep(100); + } + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); @@ -1089,6 +1180,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableMeta, updateStbMeta) { @@ -1098,7 +1190,7 @@ TEST(tableMeta, updateStbMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestSetRspDbVgroupsAndSuperMeta(); initQueryModuleMsgHandle(); @@ -1127,6 +1219,10 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(100); + } + tfree(tableMeta); STableMetaRsp rsp = {0}; @@ -1135,6 +1231,16 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n != 3) { + usleep(100); + } else { + break; + } + } + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); @@ -1157,6 +1263,7 @@ TEST(tableMeta, updateStbMeta) { tfree(tableMeta); catalogDestroy(); + memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); } @@ -1167,7 +1274,15 @@ TEST(tableDistVgroup, normalTable) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndNormalMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); initQueryModuleMsgHandle(); @@ -1191,6 +1306,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableDistVgroup, childTableCase) { @@ -1199,7 +1315,16 @@ TEST(tableDistVgroup, childTableCase) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); initQueryModuleMsgHandle(); @@ -1223,6 +1348,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(vgInfo->epset.numOfEps, 4); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(tableDistVgroup, superTableCase) { @@ -1231,7 +1357,18 @@ TEST(tableDistVgroup, superTableCase) { SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; - ctgTestSetPrepareDbVgroupsAndSuperMeta(); + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_VGINFO; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); @@ -1260,6 +1397,7 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(dbVgroup, getSetDbVgroupCase) { @@ -1272,7 +1410,14 @@ TEST(dbVgroup, getSetDbVgroupCase) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndNormalMeta(); + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + + + ctgTestSetRspByIdx(); + initQueryModuleMsgHandle(); @@ -1292,6 +1437,11 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + usleep(100); + } + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); @@ -1309,6 +1459,17 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); ASSERT_EQ(code, 0); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n != 3) { + usleep(100); + } else { + break; + } + } + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); @@ -1323,6 +1484,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, getSetRmSameDbVgroup) { @@ -1336,7 +1498,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -1374,6 +1536,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { sleep(1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } TEST(multiThread, getSetRmDiffDbVgroup) { @@ -1387,7 +1550,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroups(); + ctgTestSetRspDbVgroups(); initQueryModuleMsgHandle(); @@ -1425,6 +1588,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { sleep(1); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } @@ -1440,7 +1604,7 @@ TEST(multiThread, ctableMeta) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndChildMeta(); + ctgTestSetRspDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -1477,6 +1641,7 @@ TEST(multiThread, ctableMeta) { sleep(2); catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } @@ -1495,7 +1660,7 @@ TEST(rentTest, allRent) { ctgTestInitLogFile(); - ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); + ctgTestSetRspDbVgroupsAndMultiSuperMeta(); initQueryModuleMsgHandle(); @@ -1525,6 +1690,10 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { + usleep(100); + } + code = catalogGetExpiredDBs(pCtg, &dbs, &num); ASSERT_EQ(code, 0); printf("%d - expired dbNum:%d\n", i, num); @@ -1550,6 +1719,7 @@ TEST(rentTest, allRent) { } catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } int main(int argc, char **argv) { From 324c5742fe15699ac6a45f879605fed43569812e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Feb 2022 21:11:21 +0800 Subject: [PATCH 16/29] TD-13476 add sync.h --- include/libs/sync/sync.h | 134 ++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 283604508f..00ba1120e7 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 TAOS Data, Inc. + * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -23,7 +23,7 @@ extern "C" { #include #include "taosdef.h" -typedef int32_t SyncNodeId; +typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; typedef int64_t SyncIndex; typedef uint64_t SyncTerm; @@ -46,109 +46,113 @@ typedef struct { } SNodeInfo; typedef struct { - int32_t selfIndex; - int32_t replica; + int32_t replicaNum; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; -} SSyncCluster; +} SSyncCfg; typedef struct { - int32_t selfIndex; - int32_t replica; - SNodeInfo node[TSDB_MAX_REPLICA]; + int32_t replicaNum; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; ESyncState role[TSDB_MAX_REPLICA]; } SNodesRole; +// abstract definition of snapshot +typedef struct SSnapshot { + void* data; + SyncIndex lastApplyIndex; +} SSnapshot; + typedef struct SSyncFSM { - void* pData; + void* data; - // apply committed log, bufs will be free by sync module - int32_t (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData); + // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result + // user can do something according to the code and isWeak. for example, write data into tsdb + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // cluster commit callback - int32_t (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData); + // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result + // user can do something according to the code and isWeak. for example, write data into tsdb + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // fsm return snapshot in ppBuf, bufs will be free by sync module - // TODO: getSnapshot SHOULD be async? - int32_t (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int32_t* objId, bool* isLast); + // when log entry is updated by a new one, FpRollBackCb is called + // user can do something to roll back. for example, delete data from tsdb, or just ignore it + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code); - // fsm apply snapshot with pBuf data - int32_t (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int32_t objId, bool isLast); + // user should implement this function, use "data" to take snapshot into "snapshot" + int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); - // call when restore snapshot and log done - int32_t (*onRestoreDone)(struct SSyncFSM* fsm); - - void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf); - - void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole); + // user should implement this function, restore "data" from "snapshot" + int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot); } SSyncFSM; +// abstract definition of log store in raft +// SWal implements it typedef struct SSyncLogStore { - void* pData; + void* data; - // write log with given index - int32_t (*logWrite)(struct SSyncLogStore* logStore, SyncIndex index, SSyncBuffer* pBuf); + // append one log entry + int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); - /** - * read log from given index(included) with limit, return the actual num in nBuf, - * pBuf will be free in sync module - **/ - int32_t (*logRead)(struct SSyncLogStore* logStore, SyncIndex index, int limit, - SSyncBuffer* pBuf, int* nBuf); + // get one log entry, user need to free pBuf->data + int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); - // mark log with given index has been commtted - int32_t (*logCommit)(struct SSyncLogStore* logStore, SyncIndex index); + // update log store commit index with "index" + int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); - // prune log before given index(not included) - int32_t (*logPrune)(struct SSyncLogStore* logStore, SyncIndex index); + // truncate log with index, entries after the given index (>index) will be deleted + int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index); - // rollback log after given index(included) - int32_t (*logRollback)(struct SSyncLogStore* logStore, SyncIndex index); + // return commit index of log + SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + + // return index of last entry + SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore); + + // return term of last entry + SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore); - // return last index of log - SyncIndex (*logLastIndex)(struct SSyncLogStore* logStore); } SSyncLogStore; -typedef struct SStateManager { - void* pData; +// raft need to persist two variables in storage: currentTerm, voteFor +typedef struct SStateMgr { + void* data; - // save serialized server state data, buffer will be free by Sync - int32_t (*saveServerState)(struct SStateManager* stateMng, const char* buffer, int n); + int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm); + int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm); - // read serialized server state data, buffer will be free by Sync - int32_t (*readServerState)(struct SStateManager* stateMng, char** ppBuffer, int* n); + int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor); + int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor); - // save serialized cluster state data, buffer will be free by Sync - void (*saveClusterState)(struct SStateManager* stateMng, const char* buffer, int n); + int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); + int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); - // read serialized cluster state data, buffer will be free by Sync - int32_t (*readClusterState)(struct SStateManager* stateMng, char** ppBuffer, int* n); -} SStateManager; +} SStateMgr; typedef struct { SyncGroupId vgId; - SyncIndex appliedIndex; - SSyncCluster syncCfg; - SSyncFSM fsm; + SSyncCfg syncCfg; SSyncLogStore logStore; - SStateManager stateManager; + SStateMgr stateManager; + SSyncFSM syncFsm; + } SSyncInfo; -struct SSyncNode; -typedef struct SSyncNode SSyncNode; +// will be defined in syncInt.h, here just for complie +typedef struct SSyncNode { +} SSyncNode; int32_t syncInit(); void syncCleanUp(); -SSyncNode* syncStart(const SSyncInfo*); -void syncReconfig(const SSyncNode*, const SSyncCluster*); -void syncStop(const SSyncNode*); +int64_t syncStart(const SSyncInfo*); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg*); -int32_t syncPropose(SSyncNode* syncNode, const SSyncBuffer* pBuf, void* pData, bool isWeak); +// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); +int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); -int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode); - -int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode); +ESyncState syncGetMyRole(int64_t rid); +void syncGetNodesRole(int64_t rid, SNodesRole*); extern int32_t sDebugFlag; From c524647b969cdef2dfdd39a0ab0ab9332b4a2ae7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Feb 2022 09:39:16 +0800 Subject: [PATCH 17/29] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f427910758..76a0bad195 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -181,7 +181,7 @@ typedef struct SCtgAction { #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.qRemainNum, 1) #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.qRemainNum, 1) -#define CTG_STAT_ADD(n) qError("done:%" PRId64, atomic_add_fetch_64(&(n), 1)) +#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) #define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 732d3ab111..34c5261c17 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -217,7 +217,7 @@ int32_t ctgPushAction(SCtgMetaAction *action) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.qlock); CTG_QUEUE_ADD(); - //CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); tsem_post(&gCtgMgmt.sem); From 26c2f31341646f77d49dfba6691926ffd3671088 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Feb 2022 10:58:33 +0800 Subject: [PATCH 18/29] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 2 ++ source/libs/catalog/src/catalog.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 76a0bad195..5857437f9d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -213,6 +213,8 @@ typedef struct SCtgAction { #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 +#define CTG_IS_LOCKED(_lock) atomic_load_32((_lock)) + #define CTG_LOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ assert(atomic_load_32((_lock)) >= 0); \ diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 34c5261c17..f1c2395de4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2431,6 +2431,10 @@ void catalogDestroy(void) { tsem_post(&gCtgMgmt.sem); + while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { + usleep(1); + } + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); SCatalog *pCtg = NULL; From 504d21553e6f00fbfa372de1c34c14498967995b Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 14 Feb 2022 13:08:29 +0800 Subject: [PATCH 19/29] feature/qnode --- source/libs/catalog/test/catalogTests.cpp | 80 +++++++++++++++++------ 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 3db85b11d0..c72a70421f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -839,7 +839,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.epset.numOfEps, 3); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { - usleep(100); + usleep(10000); } ctgTestSetRspTableMeta(); @@ -856,8 +856,13 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(100); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } } @@ -946,10 +951,16 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(100); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } } + tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -1049,10 +1060,16 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(100); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } } + ctgTestSetRspCTableMeta(); tableMeta = NULL; @@ -1070,10 +1087,16 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(100); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (2 != n) { + usleep(10000); + } else { + break; + } } + tableMeta = NULL; code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); ASSERT_EQ(code, 0); @@ -1162,17 +1185,30 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(100); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } } + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); ASSERT_EQ(code, 0); - while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) || ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)) { - usleep(100); + while (true) { + int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); + if (n || m) { + usleep(10000); + } else { + break; + } } + ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); @@ -1219,10 +1255,16 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(100); + while (true) { + uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (0 == n) { + usleep(10000); + } else { + break; + } } + tfree(tableMeta); STableMetaRsp rsp = {0}; @@ -1235,7 +1277,7 @@ TEST(tableMeta, updateStbMeta) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(100); + usleep(10000); } else { break; } @@ -1438,7 +1480,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { - usleep(100); + usleep(10000); } @@ -1463,7 +1505,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(100); + usleep(10000); } else { break; } @@ -1691,7 +1733,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { - usleep(100); + usleep(10000); } code = catalogGetExpiredDBs(pCtg, &dbs, &num); From e89dc53200a1451b28ef546443c14e47a0eced31 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Feb 2022 15:27:38 +0800 Subject: [PATCH 20/29] seralize db msg --- include/common/tmsg.h | 131 +++--- source/client/src/clientHb.c | 39 +- source/common/src/tmsg.c | 279 +++++++++--- source/dnode/mnode/impl/inc/mndDb.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 407 +++++++++++------- source/dnode/mnode/impl/src/mndProfile.c | 37 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +- source/dnode/mnode/impl/test/db/db.cpp | 101 ++--- .../dnode/mnode/impl/test/profile/profile.cpp | 26 +- source/libs/catalog/test/catalogTests.cpp | 57 +-- source/libs/qcom/src/querymsg.c | 55 +-- 11 files changed, 713 insertions(+), 429 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1bd347fc24..52fa0c28f8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -328,6 +328,26 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { return buf; } +static FORCE_INLINE int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp) { + if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; + if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; + for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1; + if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp) { + if (tDecodeI8(pDecoder, &pEp->inUse) < 0) return -1; + if (tDecodeI8(pDecoder, &pEp->numOfEps) < 0) return -1; + for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + if (tDecodeU16(pDecoder, &pEp->eps[i].port) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pEp->eps[i].fqdn) < 0) return -1; + } + return 0; +} + typedef struct { int32_t acctId; int64_t clusterId; @@ -612,6 +632,27 @@ typedef struct { int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); +typedef struct { + char db[TSDB_DB_FNAME_LEN]; + uint64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo +} SUseDbRsp; + +int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); +int32_t tDeserializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); +void tFreeSUsedbRsp(SUseDbRsp* pRsp); + +typedef struct { + SArray* pArray; // Array of SUseDbRsp +} SUseDbBatchRsp; + +int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); +int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); +void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SSyncDbReq, SCompactDbReq; @@ -841,15 +882,6 @@ typedef struct { char* data; } STagData; -typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SVgroupInfo vgroupInfo[]; -} SUseDbRsp; - /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' @@ -1535,43 +1567,30 @@ typedef struct { SArray* rsps; // SArray } SClientHbBatchRsp; -static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { - return taosIntHash_64(key, keyLen); -} +static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { return taosIntHash_64(key, keyLen); } -int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); - -int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); -void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); - - -static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { - void *pIter = taosHashIterate(info, NULL); +static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { + void* pIter = taosHashIterate(info, NULL); while (pIter != NULL) { SKv* kv = (SKv*)pIter; - tfree(kv->value); - pIter = taosHashIterate(info, pIter); } } - -static FORCE_INLINE void tFreeClientHbReq(void *pReq) { +static FORCE_INLINE void tFreeClientHbReq(void* pReq) { SClientHbReq* req = (SClientHbReq*)pReq; if (req->info) { tFreeReqKvHash(req->info); - taosHashCleanup(req->info); } } -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); -void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); +int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBatchReq* pReq); +int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { - SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; + SClientHbBatchReq* req = (SClientHbBatchReq*)pReq; if (deep) { taosArrayDestroyEx(req->reqs, tFreeClientHbReq); } else { @@ -1580,54 +1599,52 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { free(pReq); } -static FORCE_INLINE void tFreeClientKv(void *pKv) { - SKv *kv = (SKv *)pKv; +static FORCE_INLINE void tFreeClientKv(void* pKv) { + SKv* kv = (SKv*)pKv; if (kv) { tfree(kv->value); } } -static FORCE_INLINE void tFreeClientHbRsp(void *pRsp) { +static FORCE_INLINE void tFreeClientHbRsp(void* pRsp) { SClientHbRsp* rsp = (SClientHbRsp*)pRsp; if (rsp->info) taosArrayDestroyEx(rsp->info, tFreeClientKv); } - static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { - SClientHbBatchRsp *rsp = (SClientHbBatchRsp*)pRsp; + SClientHbBatchRsp* rsp = (SClientHbBatchRsp*)pRsp; taosArrayDestroyEx(rsp->rsps, tFreeClientHbRsp); } +int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); +int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); -int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); -void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); - -static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKv->key); - tlen += taosEncodeFixedI32(buf, pKv->valueLen); - tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); - return tlen; +static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) { + if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; + if (tEncodeI32(pEncoder, pKv->valueLen) < 0) return -1; + if (tEncodeCStrWithLen(pEncoder, (const char*)pKv->value, pKv->valueLen) < 0) return -1; + return 0; } -static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { - buf = taosDecodeFixedI32(buf, &pKv->key); - buf = taosDecodeFixedI32(buf, &pKv->valueLen); - buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); - return buf; +static FORCE_INLINE int32_t tDecodeSKv(SCoder* pDecoder, SKv* pKv) { + if (tDecodeI32(pDecoder, &pKv->key) < 0) return -1; + if (tDecodeI32(pDecoder, &pKv->valueLen) < 0) return -1; + pKv->value = malloc(pKv->valueLen + 1); + if (pKv->value == NULL) return -1; + if (tDecodeCStrTo(pDecoder, (char*)pKv->value) < 0) return -1; + return 0; } -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; +static FORCE_INLINE int32_t tEncodeSClientHbKey(SCoder* pEncoder, const SClientHbKey* pKey) { + if (tEncodeI32(pEncoder, pKey->connId) < 0) return -1; + if (tEncodeI32(pEncoder, pKey->hbType) < 0) return -1; + return 0; } -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; +static FORCE_INLINE int32_t tDecodeSClientHbKey(SCoder* pDecoder, SClientHbKey* pKey) { + if (tDecodeI32(pDecoder, &pKey->connId) < 0) return -1; + if (tDecodeI32(pDecoder, &pKey->hbType) < 0) return -1; + return 0; } typedef struct SMqHbVgInfo { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3e1af765b0..7920ae0658 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -30,14 +30,16 @@ static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t msgLen = 0; int32_t code = 0; - - while (msgLen < valueLen) { - SUseDbRsp *rsp = (SUseDbRsp *)((char *)value + msgLen); - rsp->vgVersion = ntohl(rsp->vgVersion); - rsp->vgNum = ntohl(rsp->vgNum); - rsp->uid = be64toh(rsp->uid); + SUseDbBatchRsp batchUseRsp = {0}; + if (tDeserializeSUseDbBatchRsp(value, valueLen, &batchUseRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); if (rsp->vgVersion < 0) { @@ -52,22 +54,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog return TSDB_CODE_TSC_OUT_OF_MEMORY; } - for (int32_t i = 0; i < rsp->vgNum; ++i) { - rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId); - rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin); - rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd); - - for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) { - rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); - } - - if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + for (int32_t j = 0; j < rsp->vgNum; ++j) { + SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); + if (taosHashPut(vgInfo.vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - } - + } + code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo); if (code) { taosHashCleanup(vgInfo.vgHash); @@ -201,9 +196,10 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code tfree(param); return -1; } + char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; - tDeserializeSClientHbBatchRsp(pMsg->pData, &pRsp); + tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); int32_t rspNum = taosArrayGetSize(pRsp.rsps); @@ -416,7 +412,7 @@ static void* hbThreadFunc(void* param) { if (pReq == NULL) { continue; } - int tlen = tSerializeSClientHbBatchReq(NULL, pReq); + int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = malloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -424,8 +420,7 @@ static void* hbThreadFunc(void* param) { hbClearReqInfo(pAppHbMgr); break; } - void *abuf = buf; - tSerializeSClientHbBatchReq(&abuf, pReq); + tSerializeSClientHbBatchReq(buf, tlen, pReq); SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 20e1da9d07..68924e1be4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -85,118 +85,156 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } -int32_t tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); +static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) { + if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1; int32_t kvNum = taosHashGetSize(pReq->info); - tlen += taosEncodeFixedI32(buf, kvNum); - SKv *kv; + if (tEncodeI32(pEncoder, kvNum) < 0) return -1; void *pIter = taosHashIterate(pReq->info, NULL); while (pIter != NULL) { - kv = pIter; - tlen += taosEncodeSKv(buf, kv); - + SKv *kv = pIter; + if (tEncodeSKv(pEncoder, kv) < 0) return -1; pIter = taosHashIterate(pReq->info, pIter); } - return tlen; + + return 0; } -void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { - buf = taosDecodeSClientHbKey(buf, &pReq->connKey); +static int32_t tDeserializeSClientHbReq(SCoder *pDecoder, SClientHbReq *pReq) { + if (tDecodeSClientHbKey(pDecoder, &pReq->connKey) < 0) return -1; - // TODO: error handling - int32_t kvNum; - buf = taosDecodeFixedI32(buf, &kvNum); + int32_t kvNum = 0; + if (tDecodeI32(pDecoder, &kvNum) < 0) return -1; if (pReq->info == NULL) { pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } + if (pReq->info == NULL) return -1; for (int32_t i = 0; i < kvNum; i++) { - SKv kv; - buf = taosDecodeSKv(buf, &kv); + SKv kv = {0}; + if (tDecodeSKv(pDecoder, &kv) < 0) return -1; taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); } - return buf; + return 0; } -int32_t tSerializeSClientHbRsp(void **buf, const SClientHbRsp *pRsp) { - int32_t tlen = 0; +static int32_t tSerializeSClientHbRsp(SCoder *pEncoder, const SClientHbRsp *pRsp) { + if (tEncodeSClientHbKey(pEncoder, &pRsp->connKey) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->status) < 0) return -1; + int32_t kvNum = taosArrayGetSize(pRsp->info); - tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); - tlen += taosEncodeFixedI32(buf, pRsp->status); - tlen += taosEncodeFixedI32(buf, kvNum); + if (tEncodeI32(pEncoder, kvNum) < 0) return -1; for (int32_t i = 0; i < kvNum; i++) { - SKv *kv = (SKv *)taosArrayGet(pRsp->info, i); - tlen += taosEncodeSKv(buf, kv); + SKv *kv = taosArrayGet(pRsp->info, i); + if (tEncodeSKv(pEncoder, kv) < 0) return -1; } - return tlen; + + return 0; } -void *tDeserializeSClientHbRsp(void *buf, SClientHbRsp *pRsp) { +static int32_t tDeserializeSClientHbRsp(SCoder *pDecoder, SClientHbRsp *pRsp) { + if (tDecodeSClientHbKey(pDecoder, &pRsp->connKey) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->status) < 0) return -1; + int32_t kvNum = 0; - buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); - buf = taosDecodeFixedI32(buf, &pRsp->status); - buf = taosDecodeFixedI32(buf, &kvNum); + if (tDecodeI32(pDecoder, &kvNum) < 0) return -1; pRsp->info = taosArrayInit(kvNum, sizeof(SKv)); + if (pRsp->info == NULL) return -1; for (int32_t i = 0; i < kvNum; i++) { SKv kv = {0}; - buf = taosDecodeSKv(buf, &kv); + tDecodeSKv(pDecoder, &kv); taosArrayPush(pRsp->info, &kv); } - return buf; + return 0; } -int32_t tSerializeSClientHbBatchReq(void **buf, const SClientHbBatchReq *pBatchReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pBatchReq->reqId); +int32_t tSerializeSClientHbBatchReq(void *buf, int32_t bufLen, const SClientHbBatchReq *pBatchReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pBatchReq->reqId) < 0) return -1; + int32_t reqNum = taosArrayGetSize(pBatchReq->reqs); - tlen += taosEncodeFixedI32(buf, reqNum); + if (tEncodeI32(&encoder, reqNum) < 0) return -1; for (int32_t i = 0; i < reqNum; i++) { SClientHbReq *pReq = taosArrayGet(pBatchReq->reqs, i); - tlen += tSerializeSClientHbReq(buf, pReq); + if (tSerializeSClientHbReq(&encoder, pReq) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSClientHbBatchReq(void *buf, SClientHbBatchReq *pBatchReq) { - buf = taosDecodeFixedI64(buf, &pBatchReq->reqId); - if (pBatchReq->reqs == NULL) { - pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); - } +int32_t tDeserializeSClientHbBatchReq(void *buf, int32_t bufLen, SClientHbBatchReq *pBatchReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - int32_t reqNum; - buf = taosDecodeFixedI32(buf, &reqNum); + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchReq->reqId) < 0) return -1; + + int32_t reqNum = 0; + if (tDecodeI32(&decoder, &reqNum) < 0) return -1; + if (pBatchReq->reqs == NULL) { + pBatchReq->reqs = taosArrayInit(reqNum, sizeof(SClientHbReq)); + } for (int32_t i = 0; i < reqNum; i++) { SClientHbReq req = {0}; - buf = tDeserializeSClientHbReq(buf, &req); + tDeserializeSClientHbReq(&decoder, &req); taosArrayPush(pBatchReq->reqs, &req); } - return buf; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSClientHbBatchRsp(void **buf, const SClientHbBatchRsp *pBatchRsp) { - int32_t tlen = 0; - int32_t sz = taosArrayGetSize(pBatchRsp->rsps); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { +int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBatchRsp *pBatchRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pBatchRsp->reqId) < 0) return -1; + if (tEncodeI64(&encoder, pBatchRsp->rspId) < 0) return -1; + + int32_t rspNum = taosArrayGetSize(pBatchRsp->rsps); + if (tEncodeI32(&encoder, rspNum) < 0) return -1; + for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp *pRsp = taosArrayGet(pBatchRsp->rsps, i); - tlen += tSerializeSClientHbRsp(buf, pRsp); + if (tSerializeSClientHbRsp(&encoder, pRsp) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSClientHbBatchRsp(void *buf, SClientHbBatchRsp *pBatchRsp) { - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp)); - for (int32_t i = 0; i < sz; i++) { +int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchRsp *pBatchRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchRsp->reqId) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchRsp->rspId) < 0) return -1; + + int32_t rspNum = 0; + if (tDecodeI32(&decoder, &rspNum) < 0) return -1; + if (pBatchRsp->rsps == NULL) { + pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbReq)); + } + for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp rsp = {0}; - buf = tDeserializeSClientHbRsp(buf, &rsp); + tDeserializeSClientHbRsp(&decoder, &rsp); taosArrayPush(pBatchRsp->rsps, &rsp); } - return buf; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { @@ -1306,4 +1344,129 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { tCoderClear(&decoder); return 0; +} + +static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; + + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(pRsp->pVgroupInfos, i); + if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1; + if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1; + if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1; + } + + return 0; +} + +int32_t tSerializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tSerializeSUseDbRspImp(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSUseDbRspImp(&encoder, pUsedbRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { + if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; + + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); + if (pRsp->pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo vgInfo = {0}; + if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1; + taosArrayPush(pRsp->pVgroupInfos, &vgInfo); + } + + return 0; +} + +int32_t tDeserializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDeserializeSUseDbRspImp(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbBatchRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp usedbRsp = {0}; + if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &usedbRsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUsedbRsp(SUseDbRsp *pRsp) { taosArrayDestroy(pRsp->pVgroupInfos); } + +void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); + tFreeSUsedbRsp(pUsedbRsp); + } + + taosArrayDestroy(pRsp->pArray); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 5a3e6ed26e..ff0095f9f4 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -26,7 +26,7 @@ int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); SDbObj *mndAcquireDb(SMnode *pMnode, char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); -int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen); +int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1a01c3852a..56d743cd5a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -634,43 +634,56 @@ UPDATE_DB_OVER: } static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SAlterDbReq *pAlter = pReq->rpcMsg.pCont; - pAlter->totalBlocks = htonl(pAlter->totalBlocks); - pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); - pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); - pAlter->daysToKeep2 = htonl(pAlter->daysToKeep2); - pAlter->fsyncPeriod = htonl(pAlter->fsyncPeriod); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SAlterDbReq alterReq = {0}; - mDebug("db:%s, start to alter", pAlter->db); + if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto ALTER_DB_OVER; + } - SDbObj *pDb = mndAcquireDb(pMnode, pAlter->db); + mDebug("db:%s, start to alter", alterReq.db); + + pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { - mError("db:%s, failed to alter since %s", pAlter->db, terrstr()); - return TSDB_CODE_MND_DB_NOT_EXIST; + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto ALTER_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto ALTER_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto ALTER_DB_OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mndSetDbCfgFromAlterDbReq(&dbObj, pAlter); + code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - mndReleaseDb(pMnode, pDb); - mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); - return code; + goto ALTER_DB_OVER; } dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndUpdateDb(pMnode, pReq, pDb, &dbObj); - mndReleaseDb(pMnode, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; - if (code != 0) { - mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); - return code; +ALTER_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { @@ -772,11 +785,18 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - int32_t rspLen = sizeof(SDropDbRsp); - SDropDbRsp *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) goto DROP_DB_OVER; - memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); - pRsp->uid = htobe64(pDb->uid); + SDropDbRsp dropRsp = {0}; + memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + dropRsp.uid = pDb->uid; + + int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto DROP_DB_OVER; + } + tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); + mndTransSetRpcRsp(pTrans, pRsp, rspLen); if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; @@ -789,35 +809,54 @@ DROP_DB_OVER: } static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SDropDbReq *pDrop = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SDropDbReq dropReq = {0}; - mDebug("db:%s, start to drop", pDrop->db); + if (tDeserializeSDropDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_DB_OVER; + } - SDbObj *pDb = mndAcquireDb(pMnode, pDrop->db); + mDebug("db:%s, start to drop", dropReq.db); + + pDb = mndAcquireDb(pMnode, dropReq.db); if (pDb == NULL) { - if (pDrop->ignoreNotExists) { - mDebug("db:%s, not exist, ignore not exist is set", pDrop->db); - return TSDB_CODE_SUCCESS; + if (dropReq.ignoreNotExists) { + code = 0; + goto DROP_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); - return -1; + goto DROP_DB_OVER; } } - int32_t code = mndDropDb(pMnode, pReq, pDb); - mndReleaseDb(pMnode, pDb); - - if (code != 0) { - mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); - return code; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto DROP_DB_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto DROP_DB_OVER; + } + + code = mndDropDb(pMnode, pReq, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to drop since %s", dropReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } -static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgList, int32_t *vgNum) { +static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { int32_t vindex = 0; SSdb *pSdb = pMnode->pSdb; @@ -828,168 +867,236 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgLis if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - SVgroupInfo *pInfo = &vgList[vindex]; - pInfo->vgId = htonl(pVgroup->vgId); - pInfo->hashBegin = htonl(pVgroup->hashBegin); - pInfo->hashEnd = htonl(pVgroup->hashEnd); - pInfo->epset.numOfEps = pVgroup->replica; + SVgroupInfo vgInfo = {0}; + vgInfo.vgId = pVgroup->vgId; + vgInfo.hashBegin = pVgroup->hashBegin; + vgInfo.hashEnd = pVgroup->hashEnd; + vgInfo.epset.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEp *pEp = &pInfo->epset.eps[gid]; + SEp *pEp = &vgInfo.epset.eps[gid]; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - pEp->port = htons(pDnode->port); + pEp->port = pDnode->port; } mndReleaseDnode(pMnode, pDnode); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - pInfo->epset.inUse = gid; + vgInfo.epset.inUse = gid; } } vindex++; + taosArrayPush(pVgList, &vgInfo); } sdbRelease(pSdb, pVgroup); } - - *vgNum = vindex; } static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SSdb *pSdb = pMnode->pSdb; - SUseDbReq *pUse = pReq->rpcMsg.pCont; - pUse->vgVersion = htonl(pUse->vgVersion); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SUseDbReq usedbReq = {0}; + SUseDbRsp usedbRsp = {0}; - SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to process use db req since %s", pUse->db, terrstr()); - return -1; + if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto USE_DB_OVER; } - int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); - SUseDbRsp *pRsp = rpcMallocCont(contLen); + pDb = mndAcquireDb(pMnode, usedbReq.db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto USE_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } + + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } + + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + if (usedbReq.vgVersion < pDb->vgVersion) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } + + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + + int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); + + pReq->pCont = pRsp; + pReq->contLen = contLen; + code = 0; + +USE_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + tFreeSUsedbRsp(&usedbRsp); + + return code; +} + +int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { + SUseDbBatchRsp batchUseRsp = {0}; + batchUseRsp.pArray = taosArrayInit(numOfDbs, sizeof(SUseDbRsp)); + if (batchUseRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - int32_t vgNum = 0; + for (int32_t i = 0; i < numOfDbs; ++i) { + SDbVgVersion *pDbVgVersion = &pDbs[i]; + pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); - if (pUse->vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); - } - - memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); - pRsp->uid = htobe64(pDb->uid); - pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vgNum); - pRsp->hashMethod = pDb->hashMethod; - - pReq->pCont = pRsp; - pReq->contLen = contLen; - mndReleaseDb(pMnode, pDb); - - return 0; -} - -int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen) { - SSdb *pSdb = pMnode->pSdb; - int32_t bufSize = num * (sizeof(SUseDbRsp) + TSDB_DEFAULT_VN_PER_DB * sizeof(SVgroupInfo)); - void *buf = malloc(bufSize); - int32_t len = 0; - int32_t contLen = 0; - int32_t bufOffset = 0; - SUseDbRsp *pRsp = NULL; - - for (int32_t i = 0; i < num; ++i) { - SDbVgVersion *db = &dbs[i]; - db->dbId = be64toh(db->dbId); - db->vgVersion = ntohl(db->vgVersion); - - len = 0; - - SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); - if (pDb == NULL) { - mInfo("db %s not exist", db->dbFName); - - len = sizeof(SUseDbRsp); - } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { - len = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); - } - - if (0 == len) { + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); + if (pDb == NULL || pDbVgVersion->vgVersion >= pDb->vgVersion) { mndReleaseDb(pMnode, pDb); - + mDebug("db:%s, no need to use db", pDbVgVersion->dbFName); continue; } - contLen += len; - - if (contLen > bufSize) { - buf = realloc(buf, contLen); + SUseDbRsp usedbRsp = {0}; + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to malloc usedb response", pDb->name); + continue; } - pRsp = (SUseDbRsp *)((char *)buf + bufOffset); - memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); - if (pDb) { - int32_t vgNum = 0; - mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); - - pRsp->uid = htobe64(pDb->uid); - pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vgNum); - pRsp->hashMethod = pDb->hashMethod; - } else { - pRsp->uid = htobe64(db->dbId); - pRsp->vgNum = htonl(0); - pRsp->hashMethod = 0; - pRsp->vgVersion = htonl(-1); - } - - bufOffset += len; + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + taosArrayPush(batchUseRsp.pArray, &usedbRsp); mndReleaseDb(pMnode, pDb); } - if (contLen > 0) { - *rsp = buf; - *rspLen = contLen; - } else { - *rsp = NULL; - tfree(buf); - *rspLen = 0; + int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUseDbBatchRsp(&batchUseRsp); + return -1; } + tSerializeSUseDbBatchRsp(pRsp, rspLen, &batchUseRsp); + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUseDbBatchRsp(&batchUseRsp); return 0; } static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SSyncDbReq *pSync = pReq->rpcMsg.pCont; - SDbObj *pDb = mndAcquireDb(pMnode, pSync->db); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SSyncDbReq syncReq = {0}; + + if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &syncReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SYNC_DB_OVER; + } + + mDebug("db:%s, start to sync", syncReq.db); + + pDb = mndAcquireDb(pMnode, syncReq.db); if (pDb == NULL) { - mError("db:%s, failed to process sync db req since %s", pSync->db, terrstr()); - return -1; + goto SYNC_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto SYNC_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto SYNC_DB_OVER; + } + + // code = mndSyncDb(); + +SYNC_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process sync db req since %s", syncReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); - return 0; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCompactDbReq *pCompact = pReq->rpcMsg.pCont; - SDbObj *pDb = mndAcquireDb(pMnode, pCompact->db); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCompactDbReq compactReq = {0}; + + if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &compactReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SYNC_DB_OVER; + } + + mDebug("db:%s, start to sync", compactReq.db); + + pDb = mndAcquireDb(pMnode, compactReq.db); if (pDb == NULL) { - mError("db:%s, failed to process compact db req since %s", pCompact->db, terrstr()); - return -1; + goto SYNC_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto SYNC_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto SYNC_DB_OVER; + } + + // code = mndSyncDb(); + +SYNC_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); - return 0; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d63ade4320..f798928d9c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -343,17 +343,21 @@ static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - char *batchReqStr = pReq->rpcMsg.pCont; + SClientHbBatchReq batchReq = {0}; - tDeserializeSClientHbBatchReq(batchReqStr, &batchReq); + if (tDeserializeSClientHbBatchReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &batchReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + SArray *pArray = batchReq.reqs; - int sz = taosArrayGetSize(pArray); + int32_t sz = taosArrayGetSize(pArray); SClientHbBatchRsp batchRsp = {0}; batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); for (int i = 0; i < sz; i++) { - SClientHbReq* pHbReq = taosArrayGet(pArray, i); + SClientHbReq *pHbReq = taosArrayGet(pArray, i); if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { int32_t kvNum = taosHashGetSize(pHbReq->info); if (NULL == pHbReq->info || kvNum <= 0) { @@ -364,13 +368,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { void *pIter = taosHashIterate(pHbReq->info, NULL); while (pIter != NULL) { - SKv* kv = pIter; - + SKv *kv = pIter; + switch (kv->key) { case HEARTBEAT_KEY_DBINFO: { - void *rspMsg = NULL; + void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateDBInfo(pMnode, (SDbVgVersion *)kv->value, kv->valueLen/sizeof(SDbVgVersion), &rspMsg, &rspLen); + mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv); @@ -378,9 +382,9 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { break; } case HEARTBEAT_KEY_STBINFO: { - void *rspMsg = NULL; + void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, (SSTableMetaVersion *)kv->value, kv->valueLen/sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv); @@ -392,7 +396,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { hbRsp.status = TSDB_CODE_MND_APP_ERROR; break; } - + pIter = taosHashIterate(pHbReq->info, pIter); } @@ -407,15 +411,14 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } taosArrayDestroyEx(pArray, tFreeClientHbReq); - int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); - void* buf = rpcMallocCont(tlen); - void* abuf = buf; - tSerializeSClientHbBatchRsp(&abuf, &batchRsp); - + int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp); + void *buf = rpcMallocCont(tlen); + tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); + int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); for (int32_t i = 0; i < rspNum; ++i) { SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i); - int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info): 0; + int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; for (int32_t n = 0; n < kvNum; ++n) { SKv *kv = taosArrayGet(rsp->info, n); tfree(kv->value); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6686c0887f..3230074add 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -542,12 +542,18 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } if (sendRsp && pTrans->rpcHandle != NULL) { + void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); + if (rpcCont != NULL) { + memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); + } + free(pTrans->rpcRsp); + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle, - .pCont = pTrans->rpcRsp, + .pCont = rpcCont, .contLen = pTrans->rpcRspLen}; rpcSendResponse(&rspMsg); pTrans->rpcHandle = NULL; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 4c09caed9c..a3efad8aa2 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -127,18 +127,20 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("master", 9); { - int32_t contLen = sizeof(SAlterDbReq); + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d1"); + alterdbReq.totalBlocks = 12; + alterdbReq.daysToKeep0 = 300; + alterdbReq.daysToKeep1 = 400; + alterdbReq.daysToKeep2 = 500; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.quorum = 2; + alterdbReq.cacheLastRow = 1; - SAlterDbReq* pReq = (SAlterDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->totalBlocks = htonl(12); - pReq->daysToKeep0 = htonl(300); - pReq->daysToKeep1 = htonl(400); - pReq->daysToKeep2 = htonl(500); - pReq->fsyncPeriod = htonl(4000); - pReq->walLevel = 2; - pReq->quorum = 2; - pReq->cacheLastRow = 1; + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -196,18 +198,20 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, "1.d1"); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; - pDrop->uid = htobe64(pDrop->uid); - EXPECT_STREQ(pDrop->db, "1.d1"); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pRsp->pCont, pRsp->contLen, &dropdbRsp); + EXPECT_STREQ(dropdbRsp.db, "1.d1"); } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); @@ -260,73 +264,74 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { uint64_t d2_uid = 0; { - int32_t contLen = sizeof(SUseDbReq); + SUseDbReq usedbReq = {0}; + strcpy(usedbReq.db, "1.d2"); + usedbReq.vgVersion = -1; - SUseDbReq* pReq = (SUseDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->vgVersion = htonl(-1); + int32_t contLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSUseDbReq(pReq, contLen, &usedbReq); SRpcMsg* pMsg = test.SendReq(TDMT_MND_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; - EXPECT_STREQ(pRsp->db, "1.d2"); - pRsp->uid = htobe64(pRsp->uid); - d2_uid = pRsp->uid; - pRsp->vgVersion = htonl(pRsp->vgVersion); - pRsp->vgNum = htonl(pRsp->vgNum); - pRsp->hashMethod = pRsp->hashMethod; - EXPECT_EQ(pRsp->vgVersion, 1); - EXPECT_EQ(pRsp->vgNum, 2); - EXPECT_EQ(pRsp->hashMethod, 1); + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pCont, pMsg->contLen, &usedbRsp); + EXPECT_STREQ(usedbRsp.db, "1.d2"); + EXPECT_EQ(usedbRsp.vgVersion, 1); + EXPECT_EQ(usedbRsp.vgNum, 2); + EXPECT_EQ(usedbRsp.hashMethod, 1); + d2_uid = usedbRsp.uid; { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); + SVgroupInfo* pInfo = (SVgroupInfo*)taosArrayGet(usedbRsp.pVgroupInfos, 0); + pInfo->vgId = pInfo->vgId; + pInfo->hashBegin = pInfo->hashBegin; + pInfo->hashEnd = pInfo->hashEnd; EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); EXPECT_EQ(pInfo->epset.inUse, 0); EXPECT_EQ(pInfo->epset.numOfEps, 1); SEp* pAddr = &pInfo->epset.eps[0]; - pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); + SVgroupInfo* pInfo = (SVgroupInfo*)taosArrayGet(usedbRsp.pVgroupInfos, 1); + pInfo->vgId = pInfo->vgId; + pInfo->hashBegin = pInfo->hashBegin; + pInfo->hashEnd = pInfo->hashEnd; EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); EXPECT_EQ(pInfo->epset.inUse, 0); EXPECT_EQ(pInfo->epset.numOfEps, 1); SEp* pAddr = &pInfo->epset.eps[0]; - pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } + + tFreeSUsedbRsp(&usedbRsp); } { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, "1.d2"); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; - pDrop->uid = htobe64(pDrop->uid); - EXPECT_STREQ(pDrop->db, "1.d2"); - EXPECT_EQ(pDrop->uid, d2_uid); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pRsp->pCont, pRsp->contLen, &dropdbRsp); + EXPECT_STREQ(dropdbRsp.db, "1.d2"); + EXPECT_EQ(dropdbRsp.uid, d2_uid); } } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index ccf13d5d66..5b7bcd47b8 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -96,35 +96,35 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) { } TEST_F(MndTestProfile, 04_HeartBeatMsg) { - SClientHbBatchReq batchReq; + SClientHbBatchReq batchReq = {0}; batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq)); SClientHbReq req = {0}; req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ}; req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - SKv kv; + SKv kv = {0}; kv.key = 123; kv.value = (void*)"bcd"; kv.valueLen = 4; taosHashPut(req.info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); taosArrayPush(batchReq.reqs, &req); - int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq); - - void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); - void* bufCopy = buf; - tSerializeSClientHbBatchReq(&bufCopy, &batchReq); + int32_t tlen = tSerializeSClientHbBatchReq(NULL, 0, &batchReq); + void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); + tSerializeSClientHbBatchReq(buf, tlen, &batchReq); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - char* pRspChar = (char*)pMsg->pCont; + SClientHbBatchRsp rsp = {0}; - tDeserializeSClientHbBatchRsp(pRspChar, &rsp); + tDeserializeSClientHbBatchRsp(pMsg->pCont, pMsg->contLen, &rsp); int sz = taosArrayGetSize(rsp.rsps); ASSERT_EQ(sz, 0); - //SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); - //EXPECT_EQ(pRsp->connKey.connId, 123); - //EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); - //EXPECT_EQ(pRsp->status, 0); + + // SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); + // EXPECT_EQ(pRsp->connKey.connId, 123); + // EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); + // EXPECT_EQ(pRsp->status, 0); #if 0 int32_t contLen = sizeof(SHeartBeatReq); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 4eeae0a244..7742d8ee22 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -101,7 +101,6 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; - rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp); ASSERT_EQ(rpcRsp.code, 0); @@ -252,39 +251,43 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { } void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - SUseDbRsp *rspMsg = NULL; // todo - - pRsp->code = 0; - pRsp->contLen = sizeof(SUseDbRsp) + ctgTestVgNum * sizeof(SVgroupInfo); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (SUseDbRsp *)pRsp->pCont; - strcpy(rspMsg->db, ctgTestDbname); - rspMsg->vgVersion = htonl(ctgTestVgVersion); + SUseDbRsp usedbRsp = {0}; + strcpy(usedbRsp.db, ctgTestDbname); + usedbRsp.vgVersion = ctgTestVgVersion; ctgTestCurrentVgVersion = ctgTestVgVersion; - rspMsg->vgNum = htonl(ctgTestVgNum); - rspMsg->hashMethod = 0; - rspMsg->uid = htobe64(ctgTestDbId); + usedbRsp.vgNum = ctgTestVgNum; + usedbRsp.hashMethod = 0; + usedbRsp.uid = ctgTestDbId; + usedbRsp.pVgroupInfos = taosArrayInit(usedbRsp.vgNum, sizeof(SVgroupInfo)); - SVgroupInfo *vg = NULL; - uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; for (int32_t i = 0; i < ctgTestVgNum; ++i) { - vg = &rspMsg->vgroupInfo[i]; - - vg->vgId = htonl(i + 1); - vg->hashBegin = htonl(i * hashUnit); - vg->hashEnd = htonl(hashUnit * (i + 1) - 1); - vg->epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vg->epset.inUse = i % vg->epset.numOfEps; - for (int32_t n = 0; n < vg->epset.numOfEps; ++n) { - SEp *addr = &vg->epset.eps[n]; - strcpy(addr->fqdn, "a0"); - addr->port = htons(n + 22); + SVgroupInfo vg = {0}; + vg.vgId = i + 1; + vg.hashBegin = i * hashUnit; + vg.hashEnd = hashUnit * (i + 1) - 1; + if (i == ctgTestVgNum - 1) { + vg.hashEnd = htonl(UINT32_MAX); } + + vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg.epset.inUse = i % vg.epset.numOfEps; + for (int32_t n = 0; n < vg.epset.numOfEps; ++n) { + SEp *addr = &vg.epset.eps[n]; + strcpy(addr->fqdn, "a0"); + addr->port = n + 22; + } + + taosArrayPush(usedbRsp.pVgroupInfos, &vg); } - vg->hashEnd = htonl(UINT32_MAX); + int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSUseDbRsp(pReq, contLen, &usedbRsp); - return; + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; } void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 3e14bfca09..4acfb6239b 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -83,78 +83,63 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms return TSDB_CODE_SUCCESS; } - -int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { +int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { return TSDB_CODE_TSC_INVALID_INPUT; } - SUseDbRsp *pRsp = (SUseDbRsp *)msg; SUseDbOutput *pOut = (SUseDbOutput *)output; - int32_t code = 0; + int32_t code = 0; - if (msgSize <= sizeof(*pRsp)) { - qError("invalid use db rsp msg size, msgSize:%d", msgSize); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + SUseDbRsp usedbRsp = {0}; + if (tDeserializeSUseDbRsp(msg, msgSize, &usedbRsp) != 0) { + qError("invalid use db rsp msg, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; } - - pRsp->vgVersion = ntohl(pRsp->vgVersion); - pRsp->vgNum = ntohl(pRsp->vgNum); - pRsp->uid = be64toh(pRsp->uid); - if (pRsp->vgNum < 0) { - qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum); + if (usedbRsp.vgNum < 0) { + qError("invalid db[%s] vgroup number[%d]", usedbRsp.db, usedbRsp.vgNum); return TSDB_CODE_TSC_INVALID_VALUE; } - int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp); - if (msgSize != expectSize) { - qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); if (NULL == pOut->dbVgroup) { qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pOut->dbId = pRsp->uid; - pOut->dbVgroup->vgVersion = pRsp->vgVersion; - pOut->dbVgroup->hashMethod = pRsp->hashMethod; - pOut->dbVgroup->vgHash = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + pOut->dbId = usedbRsp.uid; + pOut->dbVgroup->vgVersion = usedbRsp.vgVersion; + pOut->dbVgroup->hashMethod = usedbRsp.hashMethod; + pOut->dbVgroup->vgHash = + taosHashInit(usedbRsp.vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { - qError("taosHashInit %d failed", pRsp->vgNum); + qError("taosHashInit %d failed", usedbRsp.vgNum); tfree(pOut->dbVgroup); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - for (int32_t i = 0; i < pRsp->vgNum; ++i) { - pRsp->vgroupInfo[i].vgId = ntohl(pRsp->vgroupInfo[i].vgId); - pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin); - pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd); + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].epset.numOfEps; ++n) { - pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); - } - - if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { qError("taosHashPut failed"); goto _return; } } - memcpy(pOut->db, pRsp->db, sizeof(pOut->db)); + memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN); return code; _return: + taosArrayDestroy(usedbRsp.pVgroupInfos); if (pOut) { taosHashCleanup(pOut->dbVgroup->vgHash); tfree(pOut->dbVgroup); } - + return code; } From d5228ebd0741666c06850dcf0007a5f8d0232efa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 14 Feb 2022 15:37:13 +0800 Subject: [PATCH 21/29] refactor code --- source/libs/transport/inc/transComm.h | 2 +- source/libs/transport/src/transCli.c | 16 +++++--- source/libs/transport/src/transComm.c | 4 +- source/libs/transport/src/transSrv.c | 54 ++++++++++++++++++------- source/libs/transport/test/pushServer.c | 2 +- 5 files changed, 53 insertions(+), 25 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index bec0375dbe..6f8da57ee7 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -233,7 +233,7 @@ typedef struct { uv_async_t* asyncs; } SAsyncPool; -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); void transDestroyAsyncPool(SAsyncPool* pool); int transSendAsync(SAsyncPool* pool, queue* mq); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2a4a1891ed..00d9174e76 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -38,6 +38,7 @@ typedef struct SCliConn { int32_t ref; // debug and log info struct sockaddr_in addr; + struct sockaddr_in locaddr; } SCliConn; typedef struct SCliMsg { @@ -130,8 +131,9 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - tDebug("client conn %p %s received from %s:%d", conn, TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), - ntohs(conn->addr.sin_port)); + tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), + inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), + ntohs(conn->locaddr.sin_port)); if (conn->push != NULL && conn->notifyCount != 0) { (*conn->push->callback)(conn->push->arg, &rpcMsg); @@ -417,8 +419,9 @@ static void clientWrite(SCliConn* pConn) { pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("client conn %p %s is send to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("client conn %p %s is send to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { @@ -433,6 +436,9 @@ static void clientConnCb(uv_connect_t* req, int status) { int addrlen = sizeof(pConn->addr); uv_tcp_getpeername((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->addr, &addrlen); + addrlen = sizeof(pConn->locaddr); + uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); + tTrace("client conn %p create", pConn); assert(pConn->stream == req->handle); @@ -579,7 +585,7 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, clientAsyncCb); pThrd->timer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->timer); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 05b732b8cb..7aa5aa16f1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -250,9 +250,7 @@ int transDestroyBuffer(SConnBuffer* buf) { transClearBuffer(buf); } -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { - static int sz = 10; - +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); pool->index = 0; pool->nAsync = sz; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 5f4daef344..15561c184c 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -31,7 +31,8 @@ typedef struct SSrvConn { void* pTransInst; // rpc init void* ahandle; // void* hostThrd; - void* pSrvMsg; + SArray* srvMsgs; + // void* pSrvMsg; struct sockaddr_in addr; @@ -94,6 +95,7 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); +static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSrvMsg* msg); @@ -310,14 +312,19 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnWriteCb(uv_write_t* req, int status) { SSrvConn* conn = req->data; - - SSrvMsg* smsg = conn->pSrvMsg; - destroySmsg(smsg); - conn->pSrvMsg = NULL; - transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); + assert(taosArrayGetSize(conn->srvMsgs) >= 1); + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); + taosArrayRemove(conn->srvMsgs, 0); + destroySmsg(msg); + + // send second data, just use for push + if (taosArrayGetSize(conn->srvMsgs) > 0) { + msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); + uvStartSendRespInternal(msg); + } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); // @@ -361,20 +368,29 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { wb->base = msg; wb->len = len; } -static void uvStartSendResp(SSrvMsg* smsg) { - // impl + +static void uvStartSendRespInternal(SSrvMsg* smsg) { uv_buf_t wb; uvPrepareSendData(smsg, &wb); SSrvConn* pConn = smsg->pConn; uv_timer_stop(pConn->pTimer); - pConn->pSrvMsg = smsg; + // pConn->pSrvMsg = smsg; // conn->pWriter->data = smsg; uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb); - - // SRpcMsg* rpcMsg = smsg->msg; - +} +static void uvStartSendResp(SSrvMsg* smsg) { + // impl + SSrvConn* pConn = smsg->pConn; + if (taosArrayGetSize(pConn->srvMsgs) > 0) { + tDebug("server conn %p push data to client %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port)); + taosArrayPush(pConn->srvMsgs, &smsg); + return; + } + taosArrayPush(pConn->srvMsgs, &smsg); + uvStartSendRespInternal(smsg); return; } static void destroySmsg(SSrvMsg* smsg) { @@ -531,7 +547,7 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -571,6 +587,7 @@ void* workerThread(void* arg) { static SSrvConn* createConn() { SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // tTrace("conn %p created", pConn); ++pConn->ref; return pConn; @@ -585,8 +602,15 @@ static void destroyConn(SSrvConn* conn, bool clear) { return; } transDestroyBuffer(&conn->readBuf); - destroySmsg(conn->pSrvMsg); - conn->pSrvMsg = NULL; + + for (int i = 0; i < taosArrayGetSize(conn->srvMsgs); i++) { + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); + destroySmsg(msg); + } + taosArrayDestroy(conn->srvMsgs); + + // destroySmsg(conn->pSrvMsg); + // conn->pSrvMsg = NULL; if (clear) { uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index f9115d3d4f..0bcc47383b 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -77,7 +77,7 @@ void processShellMsg() { taosFreeQitem(pRpcMsg); { - sleep(1); + // sleep(1); SRpcMsg nRpcMsg = {0}; nRpcMsg.pCont = rpcMallocCont(msgSize); nRpcMsg.contLen = msgSize; From 7c4142dfd1b7965c1c482884b308a4e513bb1038 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 14 Feb 2022 16:13:29 +0800 Subject: [PATCH 22/29] refactor code --- source/libs/transport/src/trans.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 707f23113b..c3d3cfa2ab 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -30,8 +30,11 @@ void* rpcOpen(const SRpcInit* pInit) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->cfp = pInit->cfp; - // pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - pRpc->numOfThreads = pInit->numOfThreads; + if (pInit->connType == TAOS_CONN_SERVER) { + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + } else { + pRpc->numOfThreads = pInit->numOfThreads; + } pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); From 6ae1bc51dce15df61c1264a2b339b4912ec6eb65 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 14 Feb 2022 16:36:38 +0800 Subject: [PATCH 23/29] add debug info --- source/libs/transport/src/transSrv.c | 29 ++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 15561c184c..7ddeb99c9d 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -35,6 +35,7 @@ typedef struct SSrvConn { // void* pSrvMsg; struct sockaddr_in addr; + struct sockaddr_in locaddr; // SRpcMsg sendMsg; // del later @@ -265,8 +266,9 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->ref++; - tDebug("server conn %p %s received from %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -361,8 +363,9 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { if (transCompressMsg(msg, len, NULL)) { // impl later } - tDebug("server conn %p %s is sent to %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), + inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); pHead->msgLen = htonl(len); wb->base = msg; @@ -384,8 +387,8 @@ static void uvStartSendResp(SSrvMsg* smsg) { // impl SSrvConn* pConn = smsg->pConn; if (taosArrayGetSize(pConn->srvMsgs) > 0) { - tDebug("server conn %p push data to client %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port)); + tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); taosArrayPush(pConn->srvMsgs, &smsg); return; } @@ -512,13 +515,23 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tTrace("server conn %p created, fd: %d", pConn, fd); + int addrlen = sizeof(pConn->addr); if (0 != uv_tcp_getpeername(pConn->pTcp, (struct sockaddr*)&pConn->addr, &addrlen)) { tError("server conn %p failed to get peer info", pConn); destroyConn(pConn, true); - } else { - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + return; } + + addrlen = sizeof(pConn->locaddr); + if (0 != uv_tcp_getsockname(pConn->pTcp, (struct sockaddr*)&pConn->locaddr, &addrlen)) { + tError("server conn %p failed to get local info", pConn); + destroyConn(pConn, true); + return; + } + + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } else { tDebug("failed to create new connection"); destroyConn(pConn, true); From 54584fd1e1e99e2fa9792b5a100a0fa0ca5612f7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Feb 2022 17:39:08 +0800 Subject: [PATCH 24/29] seralize db msg --- source/client/src/clientHb.c | 3 - source/client/src/clientMsgHandler.c | 18 +-- source/dnode/mnode/impl/src/mndDb.c | 47 ++++--- source/dnode/mnode/impl/test/stb/stb.cpp | 8 +- source/libs/parser/inc/astToMsg.h | 2 +- source/libs/parser/src/astToMsg.c | 156 +++++++++++++++++++---- source/libs/parser/src/dCDAstProcess.c | 131 +++---------------- source/libs/qcom/src/querymsg.c | 32 ++--- 8 files changed, 207 insertions(+), 190 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7920ae0658..9e5cdee516 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,7 +28,6 @@ static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) } static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t msgLen = 0; int32_t code = 0; SUseDbBatchRsp batchUseRsp = {0}; @@ -72,8 +71,6 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (code) { return code; } - - msgLen += sizeof(SUseDbRsp) + rsp->vgNum * sizeof(SVgroupInfo); } return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 8ab1880069..0f3e4af575 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -264,9 +264,13 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg->pData; + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + SName name = {0}; - tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); + + tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; tNameGetDbName(&name, db); @@ -300,14 +304,12 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SDropDbRsp *rsp = (SDropDbRsp *)pMsg->pData; - - struct SCatalog *pCatalog = NULL; - rsp->uid = be64toh(rsp->uid); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); + struct SCatalog* pCatalog = NULL; catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - - catalogRemoveDB(pCatalog, rsp->db, rsp->uid); + catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); tsem_post(&pRequest->body.rspSem); return code; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 56d743cd5a..7c42ed5df6 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -975,30 +975,37 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); - SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); - if (pDb == NULL || pDbVgVersion->vgVersion >= pDb->vgVersion) { - mndReleaseDb(pMnode, pDb); - mDebug("db:%s, no need to use db", pDbVgVersion->dbFName); - continue; - } - SUseDbRsp usedbRsp = {0}; - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { + + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); + if (pDb == NULL) { + mDebug("db:%s, no exist", pDbVgVersion->dbFName); + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDbVgVersion->dbId; + usedbRsp.vgVersion = -1; + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + } else if (pDbVgVersion->vgVersion >= pDb->vgVersion) { + mDebug("db:%s, version not changed", pDbVgVersion->dbFName); mndReleaseDb(pMnode, pDb); - mError("db:%s, failed to malloc usedb response", pDb->name); continue; + } else { + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to malloc usedb response", pDb->name); + continue; + } + + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mndReleaseDb(pMnode, pDb); } - - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; - - taosArrayPush(batchUseRsp.pArray, &usedbRsp); - mndReleaseDb(pMnode, pDb); } int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 6d0005fd15..87df50ef0e 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -69,10 +69,12 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { } void* MndTestStb::BuildDropDbReq(const char* dbname, int32_t* pContLen) { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, dbname); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, dbname); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); *pContLen = contLen; return pReq; diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index d7b6469abb..a1e4225410 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -9,7 +9,7 @@ char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t *len, SParseContext *pCtx, SMsgBuf* pMsgBuf); char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 1fae979ee9..adcd740ef6 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -39,14 +39,11 @@ char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; - createReq.maxUsers = htonl(pAcctOpt->maxUsers); - createReq.maxDbs = htonl(pAcctOpt->maxDbs); - createReq.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - createReq.maxStreams = htonl(pAcctOpt->maxStreams); - // createReq.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - createReq.maxStorage = htobe64(pAcctOpt->maxStorage); - // createReq.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - // createReq.maxConnections = htonl(pAcctOpt->maxConnections); + createReq.maxUsers = pAcctOpt->maxUsers; + createReq.maxDbs = pAcctOpt->maxDbs; + createReq.maxTimeSeries = pAcctOpt->maxTimeSeries; + createReq.maxStreams = pAcctOpt->maxStreams; + createReq.maxStorage = pAcctOpt->maxStorage; if (pAcctOpt->stat.n == 0) { createReq.accessState = -1; @@ -147,9 +144,9 @@ static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, const char* msg2 = "invalid keep value"; const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; - pMsg->daysToKeep0 = htonl(-1); - pMsg->daysToKeep1 = htonl(-1); - pMsg->daysToKeep2 = htonl(-1); + pMsg->daysToKeep0 = -1; + pMsg->daysToKeep1 = -1; + pMsg->daysToKeep2 = -1; SArray* pKeep = pCreateDb->keep; if (pKeep != NULL) { @@ -209,13 +206,13 @@ static int32_t setTimePrecision(SCreateDbReq* pMsg, const SCreateDbInfo* pCreate } static void doSetDbOptions(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb) { - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); - pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); - pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); - pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); + pMsg->cacheBlockSize = pCreateDb->cacheBlockSize; + pMsg->totalBlocks = pCreateDb->numOfBlocks; + pMsg->daysPerFile = pCreateDb->daysPerFile; + pMsg->commitTime = (int32_t)pCreateDb->commitTime; + pMsg->minRows = pCreateDb->minRowsPerBlock; + pMsg->maxRows = pCreateDb->maxRowsPerBlock; + pMsg->fsyncPeriod = pCreateDb->fsyncPeriod; pMsg->compression = (int8_t)pCreateDb->compressionLevel; pMsg->walLevel = (char)pCreateDb->walLevel; pMsg->replications = pCreateDb->replica; @@ -223,7 +220,7 @@ static void doSetDbOptions(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->update = pCreateDb->update; pMsg->cacheLastRow = pCreateDb->cachelast; - pMsg->numOfVgroups = htonl(pCreateDb->numOfVgroups); + pMsg->numOfVgroups = pCreateDb->numOfVgroups; } int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { @@ -240,12 +237,104 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext* pCtx, SMsgBuf* pMsgBuf) { - SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); - if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { - tfree(pCreateMsg); - terrno = TSDB_CODE_TSC_INVALID_OPERATION; +// can only perform the parameters based on the macro definitation +static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { + char msg[512] = {0}; + if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->replications != -1 && + (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t blocks = pCreate->totalBlocks; + if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, + TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->quorum != -1 && + (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, + TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t val = pCreate->daysPerFile; + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { + snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->cacheBlockSize; + if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { + snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, + TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && + pCreate->precision != TSDB_TIME_PRECISION_NANO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->commitTime; + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { + snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->fsyncPeriod; + if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { + snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, + TSDB_MAX_FSYNC_PERIOD); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->compression != -1 && + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->numOfVgroups; + if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { + snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); + } + + val = pCreate->maxRows; + if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) { + snprintf(msg, tListLen(msg), "invalid number of max rows in file block for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); + } + + val = pCreate->minRows; + if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) { + snprintf(msg, tListLen(msg), "invalid number of min rows in file block for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + } + + return TSDB_CODE_SUCCESS; +} + +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* len, SParseContext* pCtx, SMsgBuf* pMsgBuf) { + SCreateDbReq createReq = {0}; + + if (setDbOptions(&createReq, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_OPERATION; return NULL; } @@ -256,8 +345,23 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext* pCtx return NULL; } - tNameGetFullDbName(&name, pCreateMsg->db); - return pCreateMsg; + tNameGetFullDbName(&name, createReq.db); + + if (doCheckDbOptions(&createReq, pMsgBuf) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + return NULL; + } + + int32_t tlen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSCreateDbReq(pReq, tlen, &createReq); + *len = tlen; + return pReq; } char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 4b70d2c6c3..1a933f5d85 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -125,99 +125,6 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out return TSDB_CODE_SUCCESS; } -// can only perform the parameters based on the macro definitation -static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { - char msg[512] = {0}; - - if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->replications != -1 && - (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - int32_t blocks = ntohl(pCreate->totalBlocks); - if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, - TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->quorum != -1 && - (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, - TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { - snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, - TSDB_MAX_DAYS_PER_FILE); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->cacheBlockSize); - if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { - snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && - pCreate->precision != TSDB_TIME_PRECISION_NANO) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, - TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { - snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, - TSDB_MAX_COMMIT_TIME); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->fsyncPeriod); - if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { - snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, - TSDB_MAX_FSYNC_PERIOD); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->numOfVgroups); - if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { - snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); - } - - val = htonl(pCreate->maxRows); - if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) { - snprintf(msg, tListLen(msg), "invalid number of max rows in file block for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); - } - - val = htonl(pCreate->minRows); - if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) { - snprintf(msg, tListLen(msg), "invalid number of min rows in file block for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - } - - return TSDB_CODE_SUCCESS; -} - static int32_t validateTableColumns(SArray* pFieldList, int32_t maxRowLength, int32_t maxColumns, SMsgBuf* pMsgBuf) { const char* msg2 = "row length exceeds max length"; const char* msg3 = "duplicated column names"; @@ -852,11 +759,15 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SUseDbReq* pUseDbMsg = (SUseDbReq*)calloc(1, sizeof(SUseDbReq)); - tNameExtractFullName(&n, pUseDbMsg->db); + SUseDbReq usedbReq = {0}; + tNameExtractFullName(&n, usedbReq.db); - pDcl->pMsg = (char*)pUseDbMsg; - pDcl->msgLen = sizeof(SUseDbReq); + int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void* pBuf = malloc(bufLen); + tSerializeSUseDbReq(pBuf, bufLen, &usedbReq); + + pDcl->pMsg = pBuf; + pDcl->msgLen = bufLen; pDcl->msgType = TDMT_MND_USE_DB; break; } @@ -880,14 +791,11 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SCreateDbReq* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); - if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto _error; - } + int32_t bufLen = 0; + char* pBuf = buildCreateDbMsg(pCreateDB, &bufLen, pCtx, pMsgBuf); - pDcl->pMsg = (char*)pCreateMsg; - pDcl->msgLen = sizeof(SCreateDbReq); + pDcl->pMsg = pBuf; + pDcl->msgLen = bufLen; pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB) ? TDMT_MND_CREATE_DB : TDMT_MND_ALTER_DB; break; } @@ -905,15 +813,18 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SDropDbReq* pDropDbMsg = (SDropDbReq*)calloc(1, sizeof(SDropDbReq)); - - code = tNameExtractFullName(&name, pDropDbMsg->db); - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + SDropDbReq dropdbReq = {0}; + code = tNameExtractFullName(&name, dropdbReq.db); + dropdbReq.ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_DB_NAME_T); + int32_t bufLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pBuf = malloc(bufLen); + tSerializeSDropDbReq(pBuf, bufLen, &dropdbReq); + pDcl->msgType = TDMT_MND_DROP_DB; - pDcl->msgLen = sizeof(SDropDbReq); - pDcl->pMsg = (char*)pDropDbMsg; + pDcl->msgLen = bufLen; + pDcl->pMsg = pBuf; break; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 4acfb6239b..53ee288930 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -55,32 +55,26 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { +int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } - SBuildUseDBInput* bInput = (SBuildUseDBInput *)input; + SBuildUseDBInput *bInput = input; - int32_t estimateSize = sizeof(SUseDbReq); - if (NULL == *msg || msgSize < estimateSize) { - tfree(*msg); - *msg = rpcMallocCont(estimateSize); - if (NULL == *msg) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } + SUseDbReq usedbReq = {0}; + strncpy(usedbReq.db, bInput->db, sizeof(usedbReq.db)); + usedbReq.db[sizeof(usedbReq.db) - 1] = 0; + usedbReq.vgVersion = bInput->vgVersion; - SUseDbReq *bMsg = (SUseDbReq *)*msg; + int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSUseDbReq(pBuf, bufLen, &usedbReq); - strncpy(bMsg->db, bInput->db, sizeof(bMsg->db)); - bMsg->db[sizeof(bMsg->db) - 1] = 0; + *msg = pBuf; + *msgLen = bufLen; - bMsg->vgVersion = bInput->vgVersion; - - *msgLen = (int32_t)sizeof(*bMsg); - - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { @@ -133,7 +127,7 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { return code; _return: - taosArrayDestroy(usedbRsp.pVgroupInfos); + tFreeSUsedbRsp(&usedbRsp); if (pOut) { taosHashCleanup(pOut->dbVgroup->vgHash); From a72fe7750b556f849699da26c241bfb1c0b6c12f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Feb 2022 18:42:42 +0800 Subject: [PATCH 25/29] serialize show req --- include/common/tmsg.h | 11 ++- source/common/src/tmsg.c | 46 +++++++++++- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 14 ++-- source/dnode/mnode/impl/src/mndShow.c | 47 ++++++------ source/dnode/mnode/impl/test/acct/acct.cpp | 9 ++- source/dnode/mnode/impl/test/show/show.cpp | 20 +++-- source/libs/parser/inc/astToMsg.h | 28 +++++-- source/libs/parser/src/astToMsg.c | 83 ++++++++++++++------- source/libs/parser/src/dCDAstProcess.c | 4 +- 9 files changed, 182 insertions(+), 80 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 52fa0c28f8..b386fb0e0d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -275,6 +275,7 @@ typedef struct { int32_t tSerializeSMCreateStbReq(void** buf, SMCreateStbReq* pReq); void* tDeserializeSMCreateStbReq(void* buf, SMCreateStbReq* pReq); +void tFreeSMCreateStbReq(SMCreateStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -888,12 +889,16 @@ typedef struct { * payloadLen is the length of payload */ typedef struct { - int8_t type; + int32_t type; char db[TSDB_DB_FNAME_LEN]; - int16_t payloadLen; - char payload[]; + int32_t payloadLen; + char* payload; } SShowReq; +int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +void tFreeSShowReq(SShowReq* pReq); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroup; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 68924e1be4..fec517bc1b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -440,6 +440,11 @@ void *tDeserializeSMCreateStbReq(void *buf, SMCreateStbReq *pReq) { return buf; } +void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { + taosArrayDestroy(pReq->pColumns); + taosArrayDestroy(pReq->pTags); +} + int32_t tSerializeSMDropStbReq(void **buf, SMDropStbReq *pReq) { int32_t tlen = 0; @@ -1469,4 +1474,43 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { } taosArrayDestroy(pRsp->pArray); -} \ No newline at end of file +} + +int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->type) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->payloadLen) < 0) return -1; + if (pReq->payloadLen > 0) { + if (tEncodeCStr(&encoder, pReq->payload) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; + if (pReq->payloadLen > 0) { + pReq->payload = malloc(pReq->payloadLen); + if (pReq->payload == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSShowReq(SShowReq *pReq) { free(pReq->payload); } diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index dc8390fba3..7d17be3382 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -80,12 +80,16 @@ SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { } void Testbase::SendShowMetaReq(int8_t showType, const char* db) { - int32_t contLen = sizeof(SShowReq); - SShowReq* pShow = (SShowReq*)rpcMallocCont(contLen); - pShow->type = showType; - strcpy(pShow->db, db); + SShowReq showReq = {0}; + showReq.type = showType; + //strcpy(showReq.db, db); - SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pShow, contLen); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + char* pReq = (char*)rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); + + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pReq, contLen); SShowRsp* pShowRsp = (SShowRsp*)pRsp->pCont; ASSERT(pShowRsp != nullptr); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index e2ddcee0e9..31e3f539ff 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -118,27 +118,28 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { static int32_t mndProcessShowReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowReq *pShowReq = pReq->rpcMsg.pCont; - int8_t type = pShowReq->type; - int16_t payloadLen = htonl(pShowReq->payloadLen); + int32_t code = -1; + SShowReq showReq = {0}; - if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { - terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta req since %s", terrstr()); - return -1; + if (tDeserializeSShowReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &showReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SHOW_OVER; } - ShowMetaFp metaFp = pMgmt->metaFps[type]; + if (showReq.type <= TSDB_MGMT_TABLE_START || showReq.type >= TSDB_MGMT_TABLE_MAX) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + goto SHOW_OVER; + } + + ShowMetaFp metaFp = pMgmt->metaFps[showReq.type]; if (metaFp == NULL) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); - return -1; + goto SHOW_OVER; } - SShowObj *pShow = mndCreateShowObj(pMnode, pShowReq); + SShowObj *pShow = mndCreateShowObj(pMnode, &showReq); if (pShow == NULL) { - mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); - return -1; + goto SHOW_OVER; } int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; @@ -146,26 +147,30 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { if (pRsp == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to process show-meta req:%s since malloc rsp error", pShow->id, - mndShowStr(type)); - return -1; + goto SHOW_OVER; } - int32_t code = (*metaFp)(pReq, pShow, &pRsp->tableMeta); - mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d type:%s, result:%s", pShow->id, pShow->numOfRows, - pShow->numOfColumns, mndShowStr(type), tstrerror(code)); + code = (*metaFp)(pReq, pShow, &pRsp->tableMeta); + mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d showReq.type:%s, result:%s", pShow->id, + pShow->numOfRows, pShow->numOfColumns, mndShowStr(showReq.type), tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { pReq->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; pReq->pCont = pRsp; pRsp->showId = htobe64(pShow->id); mndReleaseShowObj(pShow, false); - return TSDB_CODE_SUCCESS; } else { rpcFreeCont(pRsp); mndReleaseShowObj(pShow, true); - return code; } + +SHOW_OVER: + if (code != 0) { + mError("failed to process show-meta req since %s", terrstr()); + } + + tFreeSShowReq(&showReq); + return code; } static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 315f23f798..2bc4f40614 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -56,10 +56,13 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { } TEST_F(MndTestAcct, 04_Show_Acct) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_ACCT; - SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); - pReq->type = TSDB_MGMT_TABLE_ACCT; + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index bc31630ffe..e7e17d65c6 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -26,11 +26,13 @@ class MndTestShow : public ::testing::Test { Testbase MndTestShow::test; TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_MAX; - SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); - pReq->type = TSDB_MGMT_TABLE_MAX; - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -38,11 +40,13 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { } TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_START; - SShowReq* pReq = (SShowReq*)rpcMallocCont(sizeof(SShowReq)); - pReq->type = TSDB_MGMT_TABLE_START; - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index a1e4225410..77d6900acf 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -1,18 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef TDENGINE_ASTTOMSG_H #define TDENGINE_ASTTOMSG_H #include "parserInt.h" #include "tmsg.h" - char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t *len, SParseContext *pCtx, SMsgBuf* pMsgBuf); -char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); -char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); +char* buildShowMsg(SShowInfo* pShowInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf); +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf); +char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index adcd740ef6..19d1569669 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "astGenerator.h" #include "parserInt.h" #include "parserUtil.h" @@ -11,7 +26,7 @@ char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, createReq.superUser = (int8_t)pUser->type; if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - // pMsg->privilege = (char)pCmd->count; + // pMsg->privilege = (char)pCmd->count; } else { strncpy(createReq.pass, pUser->passwd.z, pUser->passwd.n); } @@ -71,7 +86,7 @@ char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, return pReq; } -char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { +char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgBufLen) { SDropUserReq dropReq = {0}; SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -89,30 +104,26 @@ char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* msgLen, int64_t id, char* msgBu } tSerializeSDropUserReq(pReq, tlen, &dropReq); - *msgLen = tlen; + *outputLen = tlen; return pReq; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pCtx, SMsgBuf* pMsgBuf) { - SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); - if (pShowMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return pShowMsg; - } +char* buildShowMsg(SShowInfo* pShowInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf) { + SShowReq showReq = {.type = pShowInfo->showType}; - pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { SToken* pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); + showReq.payloadLen = pPattern->n; + showReq.payload = malloc(showReq.payloadLen); + strncpy(showReq.payload, pPattern->z, pPattern->n); } } else { SToken* pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); + showReq.payloadLen = pEpAddr->n; + showReq.payload = malloc(showReq.payloadLen); + strncpy(showReq.payload, pEpAddr->z, pEpAddr->n); } if (pShowInfo->showType == TSDB_MGMT_TABLE_STB || pShowInfo->showType == TSDB_MGMT_TABLE_VGROUP) { @@ -121,22 +132,32 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pCtx, SMsgBuf* pMsgB if (pShowInfo->prefix.n > 0) { if (pShowInfo->prefix.n >= TSDB_DB_FNAME_LEN) { terrno = buildInvalidOperationMsg(pMsgBuf, "prefix name is too long"); - tfree(pShowMsg); + tFreeSShowReq(&showReq); return NULL; } tNameSetDbName(&n, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); } else if (pCtx->db == NULL || strlen(pCtx->db) == 0) { terrno = buildInvalidOperationMsg(pMsgBuf, "database is not specified"); - tfree(pShowMsg); + tFreeSShowReq(&showReq); return NULL; } else { tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db)); } - tNameGetFullDbName(&n, pShowMsg->db); + tNameGetFullDbName(&n, showReq.db); } - return pShowMsg; + int32_t tlen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSShowReq(pReq, tlen, &showReq); + tFreeSShowReq(&showReq); + *outputLen = tlen; + return pReq; } static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { @@ -330,7 +351,7 @@ static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } -char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* len, SParseContext* pCtx, SMsgBuf* pMsgBuf) { +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf) { SCreateDbReq createReq = {0}; if (setDbOptions(&createReq, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -360,11 +381,12 @@ char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* len, SParseContext } tSerializeSCreateDbReq(pReq, tlen, &createReq); - *len = tlen; + *outputLen = tlen; return pReq; } -char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SParseContext* pParseCtx, + SMsgBuf* pMsgBuf) { SMCreateStbReq createReq = {0}; createReq.igExists = pCreateTableSql->existCheck ? 1 : 0; createReq.pColumns = pCreateTableSql->colInfo.pColumns; @@ -374,11 +396,13 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCo SName n = {0}; if (createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf) != 0) { + tFreeSMCreateStbReq(&createReq); return NULL; } if (tNameExtractFullName(&n, createReq.name) != 0) { buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); + tFreeSMCreateStbReq(&createReq); return NULL; } @@ -391,11 +415,12 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCo void* pBuf = pReq; tSerializeSMCreateStbReq(&pBuf, &createReq); - *len = tlen; + tFreeSMCreateStbReq(&createReq); + *outputLen = tlen; return pReq; } -char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; @@ -420,11 +445,11 @@ char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx void* pBuf = pReq; tSerializeSMDropStbReq(&pBuf, &dropReq); - *len = tlen; + *outputLen = tlen; return pReq; } -char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid host name (name too long, maximum length 128)"; const char* msg2 = "dnode name can not be string"; const char* msg3 = "port should be an integer that is less than 65535 and greater than 0"; @@ -469,11 +494,11 @@ char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { } tSerializeSCreateDnodeReq(pReq, tlen, &createReq); - *len = tlen; + *outputLen = tlen; return pReq; } -char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf) { SDropDnodeReq dropReq = {0}; SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -494,6 +519,6 @@ char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { } tSerializeSDropDnodeReq(pReq, tlen, &dropReq); - *len = tlen; + *outputLen = tlen; return pReq; } \ No newline at end of file diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 1a933f5d85..6283b56754 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -114,12 +114,10 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out } *pEpSet = pCtx->mgmtEpSet; - *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf); + *output = buildShowMsg(pShowInfo, outputLen, pCtx, pMsgBuf); if (*output == NULL) { return terrno; } - - *outputLen = sizeof(SShowReq) /* + htons(pShowMsg->payloadLen)*/; } return TSDB_CODE_SUCCESS; From 71826d7d2dc75cd625fec2a7ea3c9c280ba1a330 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Feb 2022 18:53:18 +0800 Subject: [PATCH 26/29] minor changes --- source/libs/parser/src/astToMsg.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 19d1569669..2bbf9116c2 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -396,13 +396,11 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SP SName n = {0}; if (createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf) != 0) { - tFreeSMCreateStbReq(&createReq); return NULL; } if (tNameExtractFullName(&n, createReq.name) != 0) { buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); - tFreeSMCreateStbReq(&createReq); return NULL; } @@ -415,7 +413,6 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SP void* pBuf = pReq; tSerializeSMCreateStbReq(&pBuf, &createReq); - tFreeSMCreateStbReq(&createReq); *outputLen = tlen; return pReq; } From 36340eb53bd16317bde73306345d2cddb9540ff2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Feb 2022 19:03:37 +0800 Subject: [PATCH 27/29] feature/qnode --- source/libs/catalog/test/catalogTests.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index c72a70421f..769be97f7d 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -809,6 +809,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } +#if 0 TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; @@ -1764,6 +1765,8 @@ TEST(rentTest, allRent) { memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } +#endif + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From fb34af9106cb5a348e09395b1b8e9653dc76b6a4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Feb 2022 21:00:39 +0800 Subject: [PATCH 28/29] serialize retrieve request --- source/client/src/clientMsgHandler.c | 15 ++++++++------- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 12 +++++++----- source/dnode/mnode/impl/src/mndShow.c | 13 ++++++++----- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 0f3e4af575..16b017b0c5 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -97,14 +97,15 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { - SRetrieveTableReq* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableReq)); - if (pRetrieveMsg == NULL) { - return NULL; - } + SRetrieveTableReq retrieveReq = {0}; + retrieveReq.showId = pRequest->body.showInfo.execId; - pRetrieveMsg->showId = htobe64(pRequest->body.showInfo.execId); - pMsgSendInfo->msgInfo.pData = pRetrieveMsg; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); + void* pReq = malloc(contLen); + tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); + + pMsgSendInfo->msgInfo.pData = pReq; + pMsgSendInfo->msgInfo.len = contLen; } else { SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 7d17be3382..e91fbd2fb7 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -132,13 +132,15 @@ int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } const char* Testbase::GetMetaTbName() { return pMeta->tbName; } void Testbase::SendShowRetrieveReq() { - int32_t contLen = sizeof(SRetrieveTableReq); + SRetrieveTableReq retrieveReq = {0}; + retrieveReq.showId = showId; + retrieveReq.free = 0; - SRetrieveTableReq* pRetrieve = (SRetrieveTableReq*)rpcMallocCont(contLen); - pRetrieve->showId = htobe64(showId); - pRetrieve->free = 0; + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); - SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pRetrieve, contLen); + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pReq, contLen); pRetrieveRsp = (SRetrieveTableRsp*)pRsp->pCont; pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 31e3f539ff..3b470fc769 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -180,10 +180,13 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { int32_t size = 0; int32_t rowsRead = 0; - SRetrieveTableReq *pRetrieve = pReq->rpcMsg.pCont; - int64_t showId = htobe64(pRetrieve->showId); + SRetrieveTableReq retrieveReq = {0}; + if (tDeserializeSRetrieveTableReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - SShowObj *pShow = mndAcquireShowObj(pMnode, showId); + SShowObj *pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); @@ -207,7 +210,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { pShow->numOfReads = pShow->numOfRows; } - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsToRead = pShow->numOfRows - pShow->numOfReads; } @@ -231,7 +234,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { } // if free flag is set, client wants to clean the resources - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsRead = (*retrieveFp)(pReq, pShow, pRsp->data, rowsToRead); } From 08a0bc9b051ecb55dd492ee07a087f719af3a6db Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Feb 2022 22:43:17 +0800 Subject: [PATCH 29/29] minor changes --- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index e91fbd2fb7..61119c4308 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -82,7 +82,7 @@ SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { void Testbase::SendShowMetaReq(int8_t showType, const char* db) { SShowReq showReq = {0}; showReq.type = showType; - //strcpy(showReq.db, db); + strcpy(showReq.db, db); int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); char* pReq = (char*)rpcMallocCont(contLen);