From 40c7562470fba96443fe24e88592cfd91c6940cd Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 15 Nov 2022 16:38:51 +0800 Subject: [PATCH 001/120] enh: optimize statement for querying the number of sub tables of the super table --- include/libs/function/functionMgt.h | 1 + include/libs/nodes/nodes.h | 1 + include/libs/nodes/plannodes.h | 4 +- source/libs/function/src/builtins.c | 25 ++++- source/libs/nodes/src/nodesCodeFuncs.c | 8 +- source/libs/nodes/src/nodesMsgFuncs.c | 2 + source/libs/nodes/src/nodesUtilFuncs.c | 3 + source/libs/parser/src/parTranslater.c | 111 +++++++++++++++++++++ source/libs/parser/test/mockCatalog.cpp | 7 +- source/libs/planner/src/planLogicCreater.c | 16 +-- source/libs/planner/src/planPhysiCreater.c | 3 + source/libs/planner/src/planSpliter.c | 17 +++- source/libs/planner/test/planSysTbTest.cpp | 6 ++ 13 files changed, 186 insertions(+), 18 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 81f63537e5..9ca6a7a9fa 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -130,6 +130,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_GROUP_KEY, FUNCTION_TYPE_CACHE_LAST_ROW, FUNCTION_TYPE_CACHE_LAST, + FUNCTION_TYPE_TABLE_COUNT, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 00e896f586..b4f256e304 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -227,6 +227,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, QUERY_NODE_PHYSICAL_PLAN_PROJECT, QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f942713f5d..aa0962a40b 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -62,7 +62,8 @@ typedef enum EScanType { SCAN_TYPE_STREAM, SCAN_TYPE_TABLE_MERGE, SCAN_TYPE_BLOCK_INFO, - SCAN_TYPE_LAST_ROW + SCAN_TYPE_LAST_ROW, + SCAN_TYPE_TABLE_COUNT } EScanType; typedef struct SScanLogicNode { @@ -314,6 +315,7 @@ typedef struct SScanPhysiNode { typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode SBlockDistScanPhysiNode; +typedef SScanPhysiNode STableCountScanPhysiNode; typedef struct SLastRowScanPhysiNode { SScanPhysiNode scan; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index d3f03e8e9c..155a184074 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -475,7 +475,7 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1506,7 +1506,7 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1519,7 +1519,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); uint8_t dbPrec = pFunc->node.resType.precision; - //if (1 != numOfParams && 3 != numOfParams && 4 != numOfParams) { + // if (1 != numOfParams && 3 != numOfParams && 4 != numOfParams) { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1835,7 +1835,7 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1894,7 +1894,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2060,6 +2060,11 @@ static int32_t translateTagsPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in return TSDB_CODE_SUCCESS; } +static int32_t translateTableCountPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT}; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -3218,6 +3223,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + { + .name = "_table_count", + .type = FUNCTION_TYPE_TABLE_COUNT, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, + .translateFunc = translateTableCountPseudoColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + }, }; // clang-format on diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 172c769433..c60f08c6a8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -221,6 +221,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiTableScan"; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: return "PhysiTableSeqScan"; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + return "PhysiTableMergeScan"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return "PhysiSreamScan"; case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: @@ -229,8 +231,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiBlockDistScan"; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return "PhysiLastRowScan"; - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - return "PhysiTableMergeScan"; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: + return "PhysiTableCountScan"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -4630,6 +4632,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicPlanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return physiScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return physiLastRowScanNodeToJson(pObj, pJson); @@ -4786,6 +4789,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicPlan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return jsonToPhysiScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return jsonToPhysiLastRowScanNode(pJson, pObj); diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 2879d55167..c263e276da 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -3630,6 +3630,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = physiScanNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: @@ -3768,6 +3769,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = msgToPhysiScanNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 39d17153d0..ea02c887cf 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -493,6 +493,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SBlockDistScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return makeNode(type, sizeof(SLastRowScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: + return makeNode(type, sizeof(STableCountScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -1118,6 +1120,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: destroyScanPhysiNode((SScanPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0e5cb14208..0bd04781bb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7464,6 +7464,112 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) { return code; } +static bool isTableCountProject(SNodeList* pProjectionList) { + if (1 != LIST_LENGTH(pProjectionList)) { + return false; + } + SNode* pProj = nodesListGetNode(pProjectionList, 0); + return QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(((SFunctionNode*)pProj)->functionName, "count"); +} + +static bool isTableCountFrom(SNode* pTable) { + if (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable)) { + return false; + } + SRealTableNode* pRtable = (SRealTableNode*)pTable; + return 0 == strcmp(pRtable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && + 0 == strcmp(pRtable->table.tableName, TSDB_INS_TABLE_TABLES); +} + +static bool isTableCountCond(SNode* pCond, const char* pCol) { + if (QUERY_NODE_OPERATOR != nodeType(pCond) || OP_TYPE_EQUAL != ((SOperatorNode*)pCond)->opType) { + return false; + } + SNode* pLeft = ((SOperatorNode*)pCond)->pLeft; + SNode* pRight = ((SOperatorNode*)pCond)->pRight; + if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_VALUE == nodeType(pRight)) { + return 0 == strcmp(((SColumnNode*)pLeft)->colName, pCol); + } + if (QUERY_NODE_COLUMN == nodeType(pRight) && QUERY_NODE_VALUE == nodeType(pLeft)) { + return 0 == strcmp(((SColumnNode*)pRight)->colName, pCol); + } + return false; +} + +static bool isTableCountWhere(SNode* pWhere) { + if (NULL == pWhere || QUERY_NODE_LOGIC_CONDITION != nodeType(pWhere)) { + return false; + } + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pWhere; + if (LOGIC_COND_TYPE_AND != pLogicCond->condType || 2 != LIST_LENGTH(pLogicCond->pParameterList)) { + return false; + } + SNode* pCond1 = nodesListGetNode(pLogicCond->pParameterList, 0); + SNode* pCond2 = nodesListGetNode(pLogicCond->pParameterList, 1); + return (isTableCountCond(pCond1, "db_name") && isTableCountCond(pCond2, "stable_name")) || + (isTableCountCond(pCond1, "stable_name") && isTableCountCond(pCond2, "db_name")); +} + +static bool isTableCountQuery(const SSelectStmt* pSelect) { + if (!isTableCountProject(pSelect->pProjectionList) || !isTableCountFrom(pSelect->pFromTable) || + !isTableCountWhere(pSelect->pWhere) || NULL != pSelect->pPartitionByList || NULL != pSelect->pWindow || + NULL != pSelect->pGroupByList || NULL != pSelect->pHaving || NULL != pSelect->pRange || NULL != pSelect->pEvery || + NULL != pSelect->pFill) { + return false; + } + return true; +} + +static SNode* createTableCountPseudoColumn() { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return NULL; + } + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", "_table_count"); + return (SNode*)pFunc; +} + +static int32_t rewriteCountFuncForTableCount(SSelectStmt* pSelect) { + SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pSelect->pProjectionList, 0); + NODES_DESTORY_LIST(pFunc->pParameterList); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", "sum"); + return nodesListMakeStrictAppend(&pFunc->pParameterList, createTableCountPseudoColumn()); +} + +static const char* getNameFromCond(SLogicConditionNode* pLogicCond, const char* pCol) { + SOperatorNode* pCond1 = (SOperatorNode*)nodesListGetNode(pLogicCond->pParameterList, 0); + SOperatorNode* pCond2 = (SOperatorNode*)nodesListGetNode(pLogicCond->pParameterList, 1); + if (QUERY_NODE_COLUMN == nodeType(pCond1->pLeft) && 0 == strcmp(((SColumnNode*)pCond1->pLeft)->colName, pCol)) { + return ((SValueNode*)pCond1->pRight)->literal; + } + if (QUERY_NODE_COLUMN == nodeType(pCond1->pRight) && 0 == strcmp(((SColumnNode*)pCond1->pRight)->colName, pCol)) { + return ((SValueNode*)pCond1->pLeft)->literal; + } + if (QUERY_NODE_COLUMN == nodeType(pCond2->pLeft) && 0 == strcmp(((SColumnNode*)pCond2->pLeft)->colName, pCol)) { + return ((SValueNode*)pCond2->pRight)->literal; + } + return ((SValueNode*)pCond2->pLeft)->literal; +} + +static int32_t rewriteRealTableForTableCount(SSelectStmt* pSelect) { + STableNode* pTable = (STableNode*)pSelect->pFromTable; + snprintf(pTable->dbName, sizeof(pTable->dbName), "%s", + getNameFromCond((SLogicConditionNode*)pSelect->pWhere, "db_name")); + snprintf(pTable->tableName, sizeof(pTable->tableName), "%s", + getNameFromCond((SLogicConditionNode*)pSelect->pWhere, "stable_name")); + nodesDestroyNode(pSelect->pWhere); + pSelect->pWhere = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t rewriteTableCountQuery(SSelectStmt* pSelect) { + int32_t code = rewriteCountFuncForTableCount(pSelect); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteRealTableForTableCount(pSelect); + } + return code; +} + static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { @@ -7524,6 +7630,11 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_FLUSH_DATABASE_STMT: code = rewriteFlushDatabase(pCxt, pQuery); break; + case QUERY_NODE_SELECT_STMT: + if (isTableCountQuery((SSelectStmt*)pQuery->pRoot)) { + code = rewriteTableCountQuery((SSelectStmt*)pQuery->pRoot); + } + break; default: break; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 8f051c67a0..564594a558 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -65,9 +65,10 @@ void generateInformationSchema(MockCatalogService* mcs) { .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .done(); - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 2) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 3) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .done(); mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 2) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) @@ -248,8 +249,8 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList); } -int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, - int32_t* tableNum, int64_t* stateTs) { +int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, + int64_t* stateTs) { return 0; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 89e8a85895..89635909ee 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -203,11 +203,13 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols } if (NULL == pScanCols) { - return NULL == pScanPseudoCols - ? SCAN_TYPE_TABLE - : ((FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType) - ? SCAN_TYPE_BLOCK_INFO - : SCAN_TYPE_TABLE); + if (NULL == pScanPseudoCols) { + return SCAN_TYPE_TABLE; + } + int32_t funcType = ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType; + return FUNCTION_TYPE_BLOCK_DIST_INFO == funcType + ? SCAN_TYPE_BLOCK_INFO + : (FUNCTION_TYPE_TABLE_COUNT == funcType ? SCAN_TYPE_TABLE_COUNT : SCAN_TYPE_TABLE); } return SCAN_TYPE_TABLE; @@ -286,6 +288,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT return TSDB_CODE_SUCCESS; } +static bool needScanDefaultCol(EScanType scanType) { return SCAN_TYPE_TABLE_COUNT != scanType; } + static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) { SScanLogicNode* pScan = NULL; @@ -320,7 +324,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->hasNormalCols = true; } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && needScanDefaultCol(pScan->scanType)) { code = addDefaultScanCol(pRealTable->pMeta, &pScan->pScanCols); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 72f3d995bc..a19835be0b 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -489,6 +489,8 @@ static ENodeType getScanOperatorType(EScanType scanType) { return QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; case SCAN_TYPE_BLOCK_INFO: return QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; + case SCAN_TYPE_TABLE_COUNT: + return QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN; default: break; } @@ -623,6 +625,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: + case SCAN_TYPE_TABLE_COUNT: return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_LAST_ROW: return createLastRowScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 249ba1815d..783dce9483 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -292,6 +292,20 @@ static bool stbSplNeedSplitJoin(bool streamQuery, SJoinLogicNode* pJoin) { return true; } +static bool stbSplIsTableCountQuery(SLogicNode* pNode) { + if (1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pNode->pChildren, 0); + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pChild)) { + if (1 != LIST_LENGTH(((SLogicNode*)pChild)->pChildren)) { + return false; + } + pChild = nodesListGetNode(((SLogicNode*)pChild)->pChildren, 0); + } + return QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && SCAN_TYPE_TABLE_COUNT == ((SScanLogicNode*)pChild)->scanType; +} + static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -301,7 +315,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { case QUERY_NODE_LOGIC_PLAN_PARTITION: return streamQuery ? false : stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: - return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); + return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode) && + !stbSplIsTableCountQuery(pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return stbSplNeedSplitWindow(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_SORT: diff --git a/source/libs/planner/test/planSysTbTest.cpp b/source/libs/planner/test/planSysTbTest.cpp index 6b40e381cc..2603b3183f 100644 --- a/source/libs/planner/test/planSysTbTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -38,3 +38,9 @@ TEST_F(PlanSysTableTest, withAgg) { run("SELECT COUNT(1) FROM ins_users"); } + +TEST_F(PlanSysTableTest, tableCount) { + useDb("root", "information_schema"); + + run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test' AND stable_name = 'st1'"); +} From 2ead89e3d6530857f41baba4d9171aa9debc0e46 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 18 Nov 2022 11:53:33 +0800 Subject: [PATCH 002/120] feat: table merge scan save work --- include/libs/function/function.h | 1 + source/dnode/vnode/inc/vnode.h | 6 ++ source/dnode/vnode/src/inc/vnodeInt.h | 6 -- source/libs/executor/inc/executorimpl.h | 11 ++++ source/libs/executor/src/executil.c | 1 + source/libs/executor/src/executorimpl.c | 5 ++ source/libs/executor/src/scanoperator.c | 80 +++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 6 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 6f2a675466..d6e584cc10 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -163,6 +163,7 @@ typedef struct tExprNode { int32_t functionId; int32_t num; struct SFunctionNode *pFunctNode; + int32_t functionType; } _function; struct { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 370103c222..fcbe5e476b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -109,6 +109,12 @@ int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; +int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); + typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ac9fabf052..1ca80c7570 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -144,12 +144,6 @@ typedef struct SMetaInfo { } SMetaInfo; int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo); -typedef struct { - int64_t uid; - int64_t ctbNum; -} SMetaStbStats; -int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); - // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); int tsdbClose(STsdb** pTsdb); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b0da277cfb..6aa2a1e5d5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -549,6 +549,17 @@ typedef struct SSysTableScanInfo { SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; +typedef struct STableCountScanInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + SExprSupp pseudoSup; + SNode* pCondition; + SName name; + uint64_t suid; + uint64_t uid; + int8_t tableType; +} STableCountScanInfo; + typedef struct SBlockDistInfo { SSDataBlock* pResBlock; STsdbReader* pHandle; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d1046ff02c..d9b7682e45 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1283,6 +1283,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExprNode->_function.functionId = pFuncNode->funcId; pExprNode->_function.pFunctNode = pFuncNode; + pExprNode->_function.functionType = pFuncNode->funcType; tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 39e876800f..b1fe016d13 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2612,6 +2612,8 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, + SExecTaskInfo* pTaskInfo); int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); @@ -2802,6 +2804,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN == type) { + STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode; + pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 77d2aeba28..c42089eee5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4839,3 +4839,83 @@ _error: taosMemoryFree(pOperator); return NULL; } + +// ==================================================================================================================== +// TableCountScanOperator +static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); +static void destoryTableCountScanOperator(void* param); + +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pScanNode, + SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + + STableCountScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + + if (!pInfo || !pOperator) { + goto _error; + } + + pInfo->readHandle = *readHandle; + + if (pScanNode->pScanPseudoCols != NULL) { + SExprSupp* pSup = &pInfo->pseudoSup; + pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + } + + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + initResultSizeInfo(&pOperator->resultInfo, 1); + pInfo->pRes = createResDataBlock(pDescNode); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + tNameAssign(&pInfo->name, &pScanNode->tableName); + + pInfo->suid = pScanNode->suid; // 0 for super table, super table uid for child table + pInfo->uid = pScanNode->uid; // super table uid, or child table uid + pInfo->tableType = pScanNode->tableType; + + setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, NULL); + return pOperator; + +_error: + if (pInfo != NULL) { + destoryTableCountScanOperator(pInfo); + } + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableCountScanInfo* pInfo = pOperator->info; + + SExprSupp* pSup = &pInfo->pseudoSup; + if (pSup->numOfExprs != 1 || pSup->pExprInfo[0].pExpr->nodeType != QUERY_NODE_FUNCTION || + pSup->pExprInfo[0].pExpr->_function.functionType != FUNCTION_TYPE_TABLE_COUNT ) { + qError("%s table count scan operator invalid pseduo columns", GET_TASKID(pTaskInfo)); + } + SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); + if (pInfo->uid != 0) { + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, pInfo->uid, &stats); + int64_t ctbNum = stats.ctbNum; + //TODO: wxy about the return type bigint or int? + colDataAppend(pColInfoData, 0, (char*)&ctbNum, false); + pInfo->pRes->info.rows = 1; + } else { + //TODO: get table count in this vnode? + } + return NULL; +} + +static void destoryTableCountScanOperator(void* param) { + STableCountScanInfo* pTableCountScanInfo = param; + blockDataDestroy(pTableCountScanInfo->pRes); + + cleanupExprSupp(&pTableCountScanInfo->pseudoSup); + taosMemoryFreeClear(param); +} From 469fcd314e0a6c5b1451c7aad137b9591250ee2f Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 21 Nov 2022 18:44:04 +0800 Subject: [PATCH 003/120] enh: ins_tables optimize --- include/libs/nodes/nodes.h | 6 + source/libs/parser/inc/parUtil.h | 2 +- source/libs/parser/src/parAstParser.c | 2 +- source/libs/parser/src/parTranslater.c | 201 +++++------------- source/libs/parser/src/parUtil.c | 2 +- source/libs/parser/test/mockCatalog.cpp | 12 +- .../libs/parser/test/mockCatalogService.cpp | 136 +++++++----- source/libs/planner/src/planLogicCreater.c | 5 +- source/libs/planner/src/planOptimizer.c | 200 ++++++++++++++++- source/libs/planner/src/planPhysiCreater.c | 1 - source/libs/planner/test/planSysTbTest.cpp | 15 +- 11 files changed, 353 insertions(+), 229 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index b4f256e304..3b0fcab5fc 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -60,6 +60,12 @@ extern "C" { for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) +#define NODES_DESTORY_NODE(node) \ + do { \ + nodesDestroyNode((node)); \ + (node) = NULL; \ + } while (0) + #define NODES_DESTORY_LIST(list) \ do { \ nodesDestroyList((list)); \ diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index c53d3f9320..ce5a63f5d0 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -86,7 +86,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); -int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); +int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5aa87d780d..8c5806ebe1 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -140,7 +140,7 @@ static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pN static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) { SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt; SName name = {0}; - int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); + int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromInsTagsImpl(pCxt, &name); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9c3ff6c26d..fba40889f8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2203,22 +2203,28 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { } static bool sysTableFromVnode(const char* pTable) { - return (0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); + return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } -static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName, - SArray** pVgroupList) { +static int32_t getVnodeSysTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName, + SArray** pVgroupList) { if (0 == pTargetName->type) { return getDBVgInfoImpl(pCxt, pName, pVgroupList); } + if (0 == strcmp(pTargetName->dbname, TSDB_INFORMATION_SCHEMA_DB) || + 0 == strcmp(pTargetName->dbname, TSDB_PERFORMANCE_SCHEMA_DB)) { + pTargetName->type = 0; + return TSDB_CODE_SUCCESS; + } + if (TSDB_DB_NAME_T == pTargetName->type) { int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || TSDB_CODE_MND_DB_IN_DROPPING == code) { + // system table query should not report errors code = TSDB_CODE_SUCCESS; } return code; @@ -2235,50 +2241,44 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge } } else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || TSDB_CODE_MND_DB_IN_DROPPING == code) { + // system table query should not report errors code = TSDB_CODE_SUCCESS; } return code; } -static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { +static int32_t getVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgs, bool* pHasUserDbCond) { if (!isSelectStmt(pCxt->pCurrStmt)) { return TSDB_CODE_SUCCESS; } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SName targetName = {0}; - int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); + int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); if (TSDB_CODE_SUCCESS == code) { - code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); + code = getVnodeSysTableVgroupListImpl(pCxt, &targetName, pName, pVgs); } + *pHasUserDbCond = (0 != targetName.type); return code; } static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* vgroupList = NULL; - if (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS)) { - code = getTagsTableVgroupList(pCxt, pName, &vgroupList); - } else if ('\0' != pRealTable->qualDbName[0]) { - if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { - code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); - } - } else { - code = getDBVgInfoImpl(pCxt, pName, &vgroupList); - } + bool hasUserDbCond = false; + SArray* pVgs = NULL; + int32_t code = getVnodeSysTableVgroupList(pCxt, pName, &pVgs, &hasUserDbCond); if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && - isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) { + isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(pVgs)) { ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) { - code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) { + code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs); } if (TSDB_CODE_SUCCESS == code) { - code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); + code = toVgroupsInfo(pVgs, &pRealTable->pVgroupList); } - taosArrayDestroy(vgroupList); + taosArrayDestroy(pVgs); return code; } @@ -2303,30 +2303,39 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea } } +static int32_t setSuperTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + SArray* vgroupList = NULL; + int32_t code = getDBVgInfoImpl(pCxt, pName, &vgroupList); + if (TSDB_CODE_SUCCESS == code) { + code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); + } + taosArrayDestroy(vgroupList); + return code; +} + +static int32_t setNormalTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + if (NULL == pRealTable->pVgroupList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pRealTable->pVgroupList->numOfVgroups = 1; + return getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups); +} + static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { if (pCxt->pParseCxt->topicQuery) { return TSDB_CODE_SUCCESS; } - int32_t code = TSDB_CODE_SUCCESS; if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { - SArray* vgroupList = NULL; - code = getDBVgInfoImpl(pCxt, pName, &vgroupList); - if (TSDB_CODE_SUCCESS == code) { - code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); - } - taosArrayDestroy(vgroupList); - } else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { - code = setSysTableVgroupList(pCxt, pName, pRealTable); - } else { - pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); - if (NULL == pRealTable->pVgroupList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pRealTable->pVgroupList->numOfVgroups = 1; - code = getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups); + return setSuperTableVgroupList(pCxt, pName, pRealTable); } - return code; + + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { + return setSysTableVgroupList(pCxt, pName, pRealTable); + } + + return setNormalTableVgroupList(pCxt, pName, pRealTable); } static uint8_t getStmtPrecision(SNode* pStmt) { @@ -2360,7 +2369,6 @@ static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLE_DISTRIBUTED) && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); @@ -7467,112 +7475,6 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) { return code; } -static bool isTableCountProject(SNodeList* pProjectionList) { - if (1 != LIST_LENGTH(pProjectionList)) { - return false; - } - SNode* pProj = nodesListGetNode(pProjectionList, 0); - return QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(((SFunctionNode*)pProj)->functionName, "count"); -} - -static bool isTableCountFrom(SNode* pTable) { - if (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable)) { - return false; - } - SRealTableNode* pRtable = (SRealTableNode*)pTable; - return 0 == strcmp(pRtable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && - 0 == strcmp(pRtable->table.tableName, TSDB_INS_TABLE_TABLES); -} - -static bool isTableCountCond(SNode* pCond, const char* pCol) { - if (QUERY_NODE_OPERATOR != nodeType(pCond) || OP_TYPE_EQUAL != ((SOperatorNode*)pCond)->opType) { - return false; - } - SNode* pLeft = ((SOperatorNode*)pCond)->pLeft; - SNode* pRight = ((SOperatorNode*)pCond)->pRight; - if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_VALUE == nodeType(pRight)) { - return 0 == strcmp(((SColumnNode*)pLeft)->colName, pCol); - } - if (QUERY_NODE_COLUMN == nodeType(pRight) && QUERY_NODE_VALUE == nodeType(pLeft)) { - return 0 == strcmp(((SColumnNode*)pRight)->colName, pCol); - } - return false; -} - -static bool isTableCountWhere(SNode* pWhere) { - if (NULL == pWhere || QUERY_NODE_LOGIC_CONDITION != nodeType(pWhere)) { - return false; - } - SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pWhere; - if (LOGIC_COND_TYPE_AND != pLogicCond->condType || 2 != LIST_LENGTH(pLogicCond->pParameterList)) { - return false; - } - SNode* pCond1 = nodesListGetNode(pLogicCond->pParameterList, 0); - SNode* pCond2 = nodesListGetNode(pLogicCond->pParameterList, 1); - return (isTableCountCond(pCond1, "db_name") && isTableCountCond(pCond2, "stable_name")) || - (isTableCountCond(pCond1, "stable_name") && isTableCountCond(pCond2, "db_name")); -} - -static bool isTableCountQuery(const SSelectStmt* pSelect) { - if (!isTableCountProject(pSelect->pProjectionList) || !isTableCountFrom(pSelect->pFromTable) || - !isTableCountWhere(pSelect->pWhere) || NULL != pSelect->pPartitionByList || NULL != pSelect->pWindow || - NULL != pSelect->pGroupByList || NULL != pSelect->pHaving || NULL != pSelect->pRange || NULL != pSelect->pEvery || - NULL != pSelect->pFill) { - return false; - } - return true; -} - -static SNode* createTableCountPseudoColumn() { - SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pFunc) { - return NULL; - } - snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", "_table_count"); - return (SNode*)pFunc; -} - -static int32_t rewriteCountFuncForTableCount(SSelectStmt* pSelect) { - SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pSelect->pProjectionList, 0); - NODES_DESTORY_LIST(pFunc->pParameterList); - snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", "sum"); - return nodesListMakeStrictAppend(&pFunc->pParameterList, createTableCountPseudoColumn()); -} - -static const char* getNameFromCond(SLogicConditionNode* pLogicCond, const char* pCol) { - SOperatorNode* pCond1 = (SOperatorNode*)nodesListGetNode(pLogicCond->pParameterList, 0); - SOperatorNode* pCond2 = (SOperatorNode*)nodesListGetNode(pLogicCond->pParameterList, 1); - if (QUERY_NODE_COLUMN == nodeType(pCond1->pLeft) && 0 == strcmp(((SColumnNode*)pCond1->pLeft)->colName, pCol)) { - return ((SValueNode*)pCond1->pRight)->literal; - } - if (QUERY_NODE_COLUMN == nodeType(pCond1->pRight) && 0 == strcmp(((SColumnNode*)pCond1->pRight)->colName, pCol)) { - return ((SValueNode*)pCond1->pLeft)->literal; - } - if (QUERY_NODE_COLUMN == nodeType(pCond2->pLeft) && 0 == strcmp(((SColumnNode*)pCond2->pLeft)->colName, pCol)) { - return ((SValueNode*)pCond2->pRight)->literal; - } - return ((SValueNode*)pCond2->pLeft)->literal; -} - -static int32_t rewriteRealTableForTableCount(SSelectStmt* pSelect) { - STableNode* pTable = (STableNode*)pSelect->pFromTable; - snprintf(pTable->dbName, sizeof(pTable->dbName), "%s", - getNameFromCond((SLogicConditionNode*)pSelect->pWhere, "db_name")); - snprintf(pTable->tableName, sizeof(pTable->tableName), "%s", - getNameFromCond((SLogicConditionNode*)pSelect->pWhere, "stable_name")); - nodesDestroyNode(pSelect->pWhere); - pSelect->pWhere = NULL; - return TSDB_CODE_SUCCESS; -} - -static int32_t rewriteTableCountQuery(SSelectStmt* pSelect) { - int32_t code = rewriteCountFuncForTableCount(pSelect); - if (TSDB_CODE_SUCCESS == code) { - code = rewriteRealTableForTableCount(pSelect); - } - return code; -} - static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { @@ -7633,11 +7535,6 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_FLUSH_DATABASE_STMT: code = rewriteFlushDatabase(pCxt, pQuery); break; - case QUERY_NODE_SELECT_STMT: - if (isTableCountQuery((SSelectStmt*)pQuery->pRoot)) { - code = rewriteTableCountQuery((SSelectStmt*)pQuery->pRoot); - } - break; default: break; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e8c3f2fa8d..ed9b1f6582 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -474,7 +474,7 @@ static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicCondition return TSDB_CODE_SUCCESS; } -int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { +int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { if (NULL == pWhere) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 564594a558..935ef77b93 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -137,7 +137,7 @@ void generatePerformanceSchema(MockCatalogService* mcs) { void generateTestTables(MockCatalogService* mcs, const std::string& db) { mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) - .setVgid(1) + .setVgid(2) .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT) .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) @@ -179,9 +179,9 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) { .addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) .addTag("tag3", TSDB_DATA_TYPE_TIMESTAMP); builder.done(); - mcs->createSubTable(db, "st1", "st1s1", 1); - mcs->createSubTable(db, "st1", "st1s2", 2); - mcs->createSubTable(db, "st1", "st1s3", 1); + mcs->createSubTable(db, "st1", "st1s1", 2); + mcs->createSubTable(db, "st1", "st1s2", 3); + mcs->createSubTable(db, "st1", "st1s3", 2); } { ITableBuilder& builder = mcs->createTableBuilder(db, "st2", TSDB_SUPER_TABLE, 3, 1) @@ -191,8 +191,8 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) { .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) .addTag("jtag", TSDB_DATA_TYPE_JSON); builder.done(); - mcs->createSubTable(db, "st2", "st2s1", 1); - mcs->createSubTable(db, "st2", "st2s2", 2); + mcs->createSubTable(db, "st2", "st2s1", 2); + mcs->createSubTable(db, "st2", "st2s2", 3); } } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 95f7af435d..b3e92525e2 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -20,15 +20,18 @@ #include #include +#include "systable.h" #include "tdatablock.h" #include "tname.h" #include "ttypes.h" +using std::string; + std::unique_ptr g_mockCatalogService; class TableBuilder : public ITableBuilder { public: - virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { + virtual TableBuilder& addColumn(const string& name, int8_t type, int32_t bytes) { assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); SSchema* col = schema()->schema + (colId_ - 1); col->type = type; @@ -142,27 +145,16 @@ class MockCatalogServiceImpl { } int32_t catalogGetDBVgList(const char* pDbFName, SArray** pVgList) const { - std::string dbFName(pDbFName); - DbMetaCache::const_iterator it = meta_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1)); - if (meta_.end() == it) { - return TSDB_CODE_FAILED; + string dbName(string(pDbFName).substr(string(pDbFName).find_last_of('.') + 1)); + if (0 == dbName.compare(TSDB_INFORMATION_SCHEMA_DB) || 0 == dbName.compare(TSDB_PERFORMANCE_SCHEMA_DB)) { + return catalogGetAllDBVgList(pVgList); } - std::set vgSet; - *pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo)); - for (const auto& vgs : it->second) { - for (const auto& vg : vgs.second->vgs) { - if (0 == vgSet.count(vg.vgId)) { - taosArrayPush(*pVgList, &vg); - vgSet.insert(vg.vgId); - } - } - } - return TSDB_CODE_SUCCESS; + return catalogGetDBVgListImpl(dbName, pVgList); } int32_t catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pDbCfg) const { - std::string dbFName(pDbFName); - DbCfgCache::const_iterator it = dbCfg_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1)); + string dbFName(pDbFName); + DbCfgCache::const_iterator it = dbCfg_.find(dbFName.substr(string(pDbFName).find_last_of('.') + 1)); if (dbCfg_.end() == it) { return TSDB_CODE_FAILED; } @@ -171,7 +163,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const { + int32_t catalogGetUdfInfo(const string& funcName, SFuncInfo* pInfo) const { auto it = udf_.find(funcName); if (udf_.end() == it) { return TSDB_CODE_FAILED; @@ -236,15 +228,15 @@ class MockCatalogServiceImpl { return code; } - TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, - int32_t numOfColumns, int32_t numOfTags) { + TableBuilder& createTableBuilder(const string& db, const string& tbname, int8_t tableType, int32_t numOfColumns, + int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); meta_[db][tbname]->schema->uid = getNextId(); return *(builder_.get()); } - void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + void createSubTable(const string& db, const string& stbname, const string& tbname, int16_t vgid) { std::unique_ptr table; if (TSDB_CODE_SUCCESS != copyTableSchemaMeta(db, stbname, &table)) { throw std::runtime_error("copyTableSchemaMeta failed"); @@ -274,13 +266,13 @@ class MockCatalogServiceImpl { // string field length #define SFL 20 // string field header -#define SH(h) CA(SFL, std::string(h)) +#define SH(h) CA(SFL, string(h)) // string field #define SF(n) CA(SFL, n) // integer field length #define IFL 10 // integer field header -#define IH(i) CA(IFL, std::string(i)) +#define IH(i) CA(IFL, string(i)) // integer field #define IF(i) CA(IFL, std::to_string(i)) // split line @@ -308,7 +300,7 @@ class MockCatalogServiceImpl { int16_t numOfFields = numOfColumns + schema->tableInfo.numOfTags; for (int16_t i = 0; i < numOfFields; ++i) { const SSchema* col = schema->schema + i; - std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type)) + std::cout << SF(string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; } std::cout << std::endl; @@ -316,7 +308,7 @@ class MockCatalogServiceImpl { } } - void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) { + void createFunction(const string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) { std::shared_ptr info(new SFuncInfo); strcpy(info->name, func.c_str()); info->funcType = funcType; @@ -342,19 +334,19 @@ class MockCatalogServiceImpl { info.expr = strdup(pReq->expr); auto it = index_.find(pReq->stb); if (index_.end() == it) { - index_.insert(std::make_pair(std::string(pReq->stb), std::vector{info})); + index_.insert(std::make_pair(string(pReq->stb), std::vector{info})); } else { it->second.push_back(info); } } - void createDnode(int32_t dnodeId, const std::string& host, int16_t port) { + void createDnode(int32_t dnodeId, const string& host, int16_t port) { SEpSet epSet = {0}; addEpIntoEpSet(&epSet, host.c_str(), port); dnode_.insert(std::make_pair(dnodeId, epSet)); } - void createDatabase(const std::string& db, bool rollup, int8_t cacheLast) { + void createDatabase(const string& db, bool rollup, int8_t cacheLast) { SDbCfgInfo cfg = {0}; if (rollup) { cfg.pRetensions = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SRetention)); @@ -364,12 +356,12 @@ class MockCatalogServiceImpl { } private: - typedef std::map> TableMetaCache; - typedef std::map DbMetaCache; - typedef std::map> UdfMetaCache; - typedef std::map> IndexMetaCache; - typedef std::map DnodeCache; - typedef std::map DbCfgCache; + typedef std::map> TableMetaCache; + typedef std::map DbMetaCache; + typedef std::map> UdfMetaCache; + typedef std::map> IndexMetaCache; + typedef std::map DnodeCache; + typedef std::map DbCfgCache; uint64_t getNextId() { return id_++; } @@ -386,15 +378,15 @@ class MockCatalogServiceImpl { return pDst; } - std::string toDbname(const std::string& dbFullName) const { - std::string::size_type n = dbFullName.find("."); - if (n == std::string::npos) { + string toDbname(const string& dbFullName) const { + string::size_type n = dbFullName.find("."); + if (n == string::npos) { return dbFullName; } return dbFullName.substr(n + 1); } - std::string ttToString(int8_t tableType) const { + string ttToString(int8_t tableType) const { switch (tableType) { case TSDB_SUPER_TABLE: return "super table"; @@ -407,7 +399,7 @@ class MockCatalogServiceImpl { } } - std::string pToString(uint8_t precision) const { + string pToString(uint8_t precision) const { switch (precision) { case TSDB_TIME_PRECISION_MILLI: return "millisecond"; @@ -420,19 +412,18 @@ class MockCatalogServiceImpl { } } - std::string dtToString(int8_t type) const { return tDataTypes[type].name; } + string dtToString(int8_t type) const { return tDataTypes[type].name; } - std::string ftToString(int16_t colid, int16_t numOfColumns) const { + string ftToString(int16_t colid, int16_t numOfColumns) const { return (0 == colid ? "column" : (colid < numOfColumns ? "column" : "tag")); } - STableMeta* getTableSchemaMeta(const std::string& db, const std::string& tbname) const { + STableMeta* getTableSchemaMeta(const string& db, const string& tbname) const { std::shared_ptr table = getTableMeta(db, tbname); return table ? table->schema : nullptr; } - int32_t copyTableSchemaMeta(const std::string& db, const std::string& tbname, - std::unique_ptr* dst) const { + int32_t copyTableSchemaMeta(const string& db, const string& tbname, std::unique_ptr* dst) const { STableMeta* src = getTableSchemaMeta(db, tbname); if (nullptr == src) { return TSDB_CODE_TSC_INVALID_TABLE_NAME; @@ -446,7 +437,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SVgroupInfo* vg) const { + int32_t copyTableVgroup(const string& db, const string& tbname, SVgroupInfo* vg) const { std::shared_ptr table = getTableMeta(db, tbname); if (table->vgs.empty()) { return TSDB_CODE_SUCCESS; @@ -455,7 +446,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SArray** vgList) const { + int32_t copyTableVgroup(const string& db, const string& tbname, SArray** vgList) const { std::shared_ptr table = getTableMeta(db, tbname); if (table->vgs.empty()) { return TSDB_CODE_SUCCESS; @@ -467,7 +458,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + std::shared_ptr getTableMeta(const string& db, const string& tbname) const { DbMetaCache::const_iterator it = meta_.find(db); if (meta_.end() == it) { return std::shared_ptr(); @@ -527,6 +518,40 @@ class MockCatalogServiceImpl { return code; } + int32_t catalogGetDBVgListImpl(const string& dbName, SArray** pVgList) const { + DbMetaCache::const_iterator it = meta_.find(dbName); + if (meta_.end() == it) { + return TSDB_CODE_FAILED; + } + std::set vgSet; + *pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo)); + for (const auto& vgs : it->second) { + for (const auto& vg : vgs.second->vgs) { + if (0 == vgSet.count(vg.vgId)) { + taosArrayPush(*pVgList, &vg); + vgSet.insert(vg.vgId); + } + } + } + return TSDB_CODE_SUCCESS; + } + + int32_t catalogGetAllDBVgList(SArray** pVgList) const { + std::set vgSet; + *pVgList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVgroupInfo)); + for (const auto& db : meta_) { + for (const auto& vgs : db.second) { + for (const auto& vg : vgs.second->vgs) { + if (0 == vgSet.count(vg.vgId)) { + taosArrayPush(*pVgList, &vg); + vgSet.insert(vg.vgId); + } + } + } + } + return TSDB_CODE_SUCCESS; + } + int32_t getAllDbCfg(SArray* pDbCfgReq, SArray** pDbCfgData) const { int32_t code = TSDB_CODE_SUCCESS; if (NULL != pDbCfgReq) { @@ -634,30 +659,29 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() {} -ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, - int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { +ITableBuilder& MockCatalogService::createTableBuilder(const string& db, const string& tbname, int8_t tableType, + int32_t numOfColumns, int32_t numOfTags) { return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } -void MockCatalogService::createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, - int16_t vgid) { +void MockCatalogService::createSubTable(const string& db, const string& stbname, const string& tbname, int16_t vgid) { impl_->createSubTable(db, stbname, tbname, vgid); } void MockCatalogService::showTables() const { impl_->showTables(); } -void MockCatalogService::createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, +void MockCatalogService::createFunction(const string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) { impl_->createFunction(func, funcType, outputType, outputLen, bufSize); } void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); } -void MockCatalogService::createDnode(int32_t dnodeId, const std::string& host, int16_t port) { +void MockCatalogService::createDnode(int32_t dnodeId, const string& host, int16_t port) { impl_->createDnode(dnodeId, host, port); } -void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8_t cacheLast) { +void MockCatalogService::createDatabase(const string& db, bool rollup, int8_t cacheLast) { impl_->createDatabase(db, rollup, cacheLast); } @@ -683,7 +707,7 @@ int32_t MockCatalogService::catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pD return impl_->catalogGetDBCfg(pDbFName, pDbCfg); } -int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const { +int32_t MockCatalogService::catalogGetUdfInfo(const string& funcName, SFuncInfo* pInfo) const { return impl_->catalogGetUdfInfo(funcName, pInfo); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 89635909ee..3d1c939152 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -206,10 +206,9 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols if (NULL == pScanPseudoCols) { return SCAN_TYPE_TABLE; } - int32_t funcType = ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType; - return FUNCTION_TYPE_BLOCK_DIST_INFO == funcType + return FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType ? SCAN_TYPE_BLOCK_INFO - : (FUNCTION_TYPE_TABLE_COUNT == funcType ? SCAN_TYPE_TABLE_COUNT : SCAN_TYPE_TABLE); + : SCAN_TYPE_TABLE; } return SCAN_TYPE_TABLE; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 45fa67faef..ad0aac977d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -16,6 +16,7 @@ #include "filter.h" #include "functionMgt.h" #include "planInt.h" +#include "systable.h" #include "tglobal.h" #include "ttime.h" @@ -64,6 +65,7 @@ typedef enum ECondAction { } ECondAction; typedef bool (*FMayBeOptimized)(SLogicNode* pNode); +typedef bool (*FShouldBeOptimized)(SLogicNode* pNode, void* pInfo); static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) { if (func(pNode)) { @@ -79,6 +81,19 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) return NULL; } +static bool optFindEligibleNode(SLogicNode* pNode, FShouldBeOptimized func, void* pInfo) { + if (func(pNode, pInfo)) { + return true; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + if (optFindEligibleNode((SLogicNode*)pChild, func, pInfo)) { + return true; + } + } + return false; +} + static void optResetParent(SLogicNode* pNode) { SNode* pChild = NULL; FOREACH(pChild, pNode->pChildren) { ((SLogicNode*)pChild)->pParent = pNode; } @@ -2440,6 +2455,188 @@ static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return TSDB_CODE_SUCCESS; } +typedef struct STbCntScanOptInfo { + SAggLogicNode* pAgg; + SScanLogicNode* pScan; + SName table; +} STbCntScanOptInfo; + +static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { + if (1 != LIST_LENGTH(pAgg->pAggFuncs) || NULL != pAgg->pGroupKeys) { + return false; + } + SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0); + if (FUNCTION_TYPE_COUNT != pFunc->funcType) { + return false; + } + return true; +} + +static bool tbCntScanOptGetColValFromCond(SOperatorNode* pOper, SColumnNode** pCol, SValueNode** pVal) { + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + + *pCol = NULL; + *pVal = NULL; + if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { + *pCol = (SColumnNode*)pOper->pLeft; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { + *pVal = (SValueNode*)pOper->pLeft; + } + if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { + *pCol = (SColumnNode*)pOper->pRight; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { + *pVal = (SValueNode*)pOper->pRight; + } + + return NULL != *pCol && NULL != *pVal; +} + +static bool tbCntScanOptIsEligibleLogicCond(STbCntScanOptInfo* pInfo, SLogicConditionNode* pCond) { + if (LOGIC_COND_TYPE_AND != pCond->condType) { + return false; + } + + bool hasDbCond = false; + bool hasStbCond = false; + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + SNode* pNode = NULL; + FOREACH(pNode, pCond->pParameterList) { + if (QUERY_NODE_OPERATOR != nodeType(pNode) || !tbCntScanOptGetColValFromCond((SOperatorNode*)pNode, &pCol, &pVal)) { + return false; + } + if (!hasDbCond && 0 == strcmp(pCol->colName, "db_name")) { + hasDbCond = true; + strcpy(pInfo->table.dbname, pVal->literal); + } else if (!hasStbCond && 0 == strcmp(pCol->colName, "stable_name")) { + hasStbCond = true; + strcpy(pInfo->table.tname, pVal->literal); + } else { + return false; + } + } + return hasDbCond; +} + +static bool tbCntScanOptIsEligibleOpCond(SOperatorNode* pCond) { + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + if (!tbCntScanOptGetColValFromCond(pCond, &pCol, &pVal)) { + return false; + } + return 0 == strcmp(pCol->colName, "db_name"); +} + +static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pConditions) { + if (NULL == pConditions) { + return true; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); + } + + if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); + } + + return false; +} + +static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) { + if (0 != strcmp(pInfo->pScan->tableName.dbname, TSDB_INFORMATION_SCHEMA_DB) || + 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags) { + return false; + } + return tbCntScanOptIsEligibleConds(pInfo, pInfo->pScan->node.pConditions); +} + +static bool tbCntScanOptShouldBeOptimized(SLogicNode* pNode, STbCntScanOptInfo* pInfo) { + if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + return false; + } + + pInfo->pAgg = (SAggLogicNode*)pNode; + pInfo->pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); + return tbCntScanOptIsEligibleAgg(pInfo->pAgg) && tbCntScanOptIsEligibleScan(pInfo); +} + +static SNode* tbCntScanOptCreateTableCountFunc() { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return NULL; + } + strcpy(pFunc->functionName, "_table_count"); + strcpy(pFunc->node.aliasName, "_table_count"); + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc, NULL, 0)) { + nodesDestroyNode((SNode*)pFunc); + return NULL; + } + return (SNode*)pFunc; +} + +static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo, SScanLogicNode* pScan) { + pScan->scanType = SCAN_TYPE_TABLE_COUNT; + strcpy(pScan->tableName.dbname, pInfo->table.dbname); + strcpy(pScan->tableName.tname, pInfo->table.tname); + NODES_DESTORY_LIST(pScan->node.pTargets); + NODES_DESTORY_NODE(pScan->node.pConditions); + NODES_DESTORY_LIST(pScan->pScanCols); + NODES_DESTORY_LIST(pScan->pScanPseudoCols); + int32_t code = nodesListMakeStrictAppend(&pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc()); + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExpr(nodesListGetNode(pScan->pScanPseudoCols, 0), &pScan->node.pTargets); + } + return code; +} + +static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, SNode** pOutput) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "sum"); + strcpy(pFunc->node.aliasName, pCntFunc->node.aliasName); + int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pParam)); + if (TSDB_CODE_SUCCESS == code) { + code = fmGetFuncInfo(pFunc, NULL, 0); + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SNode*)pFunc; + } else { + nodesDestroyNode((SNode*)pFunc); + } + return code; +} + +static int32_t tbCntScanOptRewriteAgg(SAggLogicNode* pAgg) { + SNode* pSum = NULL; + int32_t code = tbCntScanOptCreateSumFunc( + (SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0), + nodesListGetNode(((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->pTargets, 0), &pSum); + if (TSDB_CODE_SUCCESS == code) { + NODES_DESTORY_LIST(pAgg->pAggFuncs); + code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, pSum); + } + return code; +} + +static int32_t tableCountScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + STbCntScanOptInfo info = {0}; + if (!optFindEligibleNode(pLogicSubplan->pNode, (FShouldBeOptimized)tbCntScanOptShouldBeOptimized, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = tbCntScanOptRewriteScan(&info, info.pScan); + if (TSDB_CODE_SUCCESS == code) { + code = tbCntScanOptRewriteAgg(info.pAgg); + } + return code; +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, @@ -2454,7 +2651,8 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}, {.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize}, {.pName = "TagScan", .optimizeFunc = tagScanOptimize}, - {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize} + {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}, + {.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize}, }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a19835be0b..f1c0812612 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -591,7 +591,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->accountId = pCxt->pPlanCxt->acctId; pScan->sysInfo = pCxt->pPlanCxt->sysInfo; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { diff --git a/source/libs/planner/test/planSysTbTest.cpp b/source/libs/planner/test/planSysTbTest.cpp index 2603b3183f..fc00285a6d 100644 --- a/source/libs/planner/test/planSysTbTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -20,13 +20,6 @@ using namespace std; class PlanSysTableTest : public PlannerTestBase {}; -TEST_F(PlanSysTableTest, show) { - useDb("root", "test"); - - run("show tables"); - run("show stables"); -} - TEST_F(PlanSysTableTest, informationSchema) { useDb("root", "information_schema"); @@ -42,5 +35,13 @@ TEST_F(PlanSysTableTest, withAgg) { TEST_F(PlanSysTableTest, tableCount) { useDb("root", "information_schema"); + run("SELECT COUNT(*) FROM ins_tables"); + + run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test'"); + run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test' AND stable_name = 'st1'"); + + run("SELECT db_name, COUNT(*) FROM ins_tables GROUP BY db_name"); + + run("SELECT db_name, stable_name, COUNT(*) FROM ins_tables GROUP BY db_name, stable_name"); } From bc180e39c997895ce1af952f0d2ca7876631a3c9 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 22 Nov 2022 16:39:30 +0800 Subject: [PATCH 004/120] enh: ins_tables optimize --- include/libs/nodes/plannodes.h | 3 +- source/libs/nodes/src/nodesCodeFuncs.c | 2 +- source/libs/planner/src/planOptimizer.c | 75 +++++++++++++++++----- source/libs/planner/src/planPhysiCreater.c | 21 +++++- 4 files changed, 82 insertions(+), 19 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index aa0962a40b..74daef4047 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -315,7 +315,6 @@ typedef struct SScanPhysiNode { typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode SBlockDistScanPhysiNode; -typedef SScanPhysiNode STableCountScanPhysiNode; typedef struct SLastRowScanPhysiNode { SScanPhysiNode scan; @@ -324,6 +323,8 @@ typedef struct SLastRowScanPhysiNode { bool ignoreNull; } SLastRowScanPhysiNode; +typedef SLastRowScanPhysiNode STableCountScanPhysiNode; + typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; SEpSet mgmtEpSet; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c60f08c6a8..e1482c8720 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -4632,9 +4632,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicPlanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return physiScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return physiLastRowScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index ad0aac977d..6355f23285 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2461,17 +2461,51 @@ typedef struct STbCntScanOptInfo { SName table; } STbCntScanOptInfo; -static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { - if (1 != LIST_LENGTH(pAgg->pAggFuncs) || NULL != pAgg->pGroupKeys) { +static bool tbCntScanOptIsEligibleGroupKeys(SNodeList* pGroupKeys) { + if (NULL == pGroupKeys) { + return true; + } + + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, pGroupKeys) { + SNode* pKey = nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0); + if (QUERY_NODE_COLUMN != nodeType(pKey)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pKey; + if (0 != strcmp(pCol->colName, "db_name") && 0 != strcmp(pCol->colName, "stable_name")) { + return false; + } + } + + return true; +} + +static bool tbCntScanOptNotNullableExpr(SNode* pNode) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } - SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0); - if (FUNCTION_TYPE_COUNT != pFunc->funcType) { - return false; + const char* pColName = ((SColumnNode*)pNode)->colName; + return 0 == strcmp(pColName, "*") || 0 == strcmp(pColName, "db_name") || 0 == strcmp(pColName, "stable_name") || + 0 == strcmp(pColName, "table_name"); +} + +static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) { + SNode* pNode = NULL; + FOREACH(pNode, pAggFuncs) { + SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pAggFuncs, 0); + if (FUNCTION_TYPE_COUNT != pFunc->funcType || + !tbCntScanOptNotNullableExpr(nodesListGetNode(pFunc->pParameterList, 0))) { + return false; + } } return true; } +static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { + return tbCntScanOptIsEligibleGroupKeys(pAgg->pGroupKeys) && tbCntScanOptIsEligibleAggFuncs(pAgg->pAggFuncs); +} + static bool tbCntScanOptGetColValFromCond(SOperatorNode* pOper, SColumnNode** pCol, SValueNode** pVal) { if (OP_TYPE_EQUAL != pOper->opType) { return false; @@ -2578,17 +2612,25 @@ static SNode* tbCntScanOptCreateTableCountFunc() { return (SNode*)pFunc; } -static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo, SScanLogicNode* pScan) { - pScan->scanType = SCAN_TYPE_TABLE_COUNT; - strcpy(pScan->tableName.dbname, pInfo->table.dbname); - strcpy(pScan->tableName.tname, pInfo->table.tname); - NODES_DESTORY_LIST(pScan->node.pTargets); - NODES_DESTORY_NODE(pScan->node.pConditions); - NODES_DESTORY_LIST(pScan->pScanCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); - int32_t code = nodesListMakeStrictAppend(&pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc()); +static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { + pInfo->pScan->scanType = SCAN_TYPE_TABLE_COUNT; + strcpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname); + strcpy(pInfo->pScan->tableName.tname, pInfo->table.tname); + NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); + NODES_DESTORY_NODE(pInfo->pScan->node.pConditions); + NODES_DESTORY_LIST(pInfo->pScan->pScanCols); + NODES_DESTORY_LIST(pInfo->pScan->pScanPseudoCols); + int32_t code = nodesListMakeStrictAppend(&pInfo->pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc()); if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExpr(nodesListGetNode(pScan->pScanPseudoCols, 0), &pScan->node.pTargets); + code = createColumnByRewriteExpr(nodesListGetNode(pInfo->pScan->pScanPseudoCols, 0), &pInfo->pScan->node.pTargets); + } + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, pInfo->pAgg->pGroupKeys) { + code = nodesListMakeStrictAppend( + &pInfo->pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + if (TSDB_CODE_SUCCESS != code) { + break; + } } return code; } @@ -2621,6 +2663,7 @@ static int32_t tbCntScanOptRewriteAgg(SAggLogicNode* pAgg) { NODES_DESTORY_LIST(pAgg->pAggFuncs); code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, pSum); } + NODES_DESTORY_LIST(pAgg->pGroupKeys); return code; } @@ -2630,7 +2673,7 @@ static int32_t tableCountScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo return TSDB_CODE_SUCCESS; } - int32_t code = tbCntScanOptRewriteScan(&info, info.pScan); + int32_t code = tbCntScanOptRewriteScan(&info); if (TSDB_CODE_SUCCESS == code) { code = tbCntScanOptRewriteAgg(info.pAgg); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index f1c0812612..1a20a80f54 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -526,6 +526,24 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu pScan->ignoreNull = pScanLogicNode->igLastNull; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); +} + +static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, + SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + STableCountScanPhysiNode* pScan = (STableCountScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags); + if (NULL != pScanLogicNode->pGroupTags && NULL == pScan->pGroupTags) { + nodesDestroyNode((SNode*)pScan); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pScan->groupSort = pScanLogicNode->groupSort; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); @@ -624,8 +642,9 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: - case SCAN_TYPE_TABLE_COUNT: return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); + case SCAN_TYPE_TABLE_COUNT: + return createTableCountScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_LAST_ROW: return createLastRowScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: From 7f6ff3964875ae508bb96a904260bdec48b33e16 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 18 Nov 2022 11:53:33 +0800 Subject: [PATCH 005/120] feat: table merge scan save work --- include/libs/function/function.h | 1 + source/dnode/vnode/inc/vnode.h | 6 ++ source/dnode/vnode/src/inc/vnodeInt.h | 6 -- source/libs/executor/inc/executorimpl.h | 11 ++++ source/libs/executor/src/executil.c | 1 + source/libs/executor/src/executorimpl.c | 5 ++ source/libs/executor/src/scanoperator.c | 80 +++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 6 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 6f2a675466..d6e584cc10 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -163,6 +163,7 @@ typedef struct tExprNode { int32_t functionId; int32_t num; struct SFunctionNode *pFunctNode; + int32_t functionType; } _function; struct { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 27084700b0..52a8a7f5a3 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -109,6 +109,12 @@ int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; +int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); + typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ac9fabf052..1ca80c7570 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -144,12 +144,6 @@ typedef struct SMetaInfo { } SMetaInfo; int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo); -typedef struct { - int64_t uid; - int64_t ctbNum; -} SMetaStbStats; -int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); - // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); int tsdbClose(STsdb** pTsdb); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f179c7bd41..1785553c39 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -530,6 +530,17 @@ typedef struct SSysTableScanInfo { SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; +typedef struct STableCountScanInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + SExprSupp pseudoSup; + SNode* pCondition; + SName name; + uint64_t suid; + uint64_t uid; + int8_t tableType; +} STableCountScanInfo; + typedef struct SBlockDistInfo { SSDataBlock* pResBlock; STsdbReader* pHandle; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d1046ff02c..d9b7682e45 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1283,6 +1283,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExprNode->_function.functionId = pFuncNode->funcId; pExprNode->_function.pFunctNode = pFuncNode; + pExprNode->_function.functionType = pFuncNode->funcType; tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0dd5765aa4..24c9244bda 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2543,6 +2543,8 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, + SExecTaskInfo* pTaskInfo); int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); @@ -2733,6 +2735,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN == type) { + STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode; + pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 916b6df969..a7298df58d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4788,3 +4788,83 @@ _error: taosMemoryFree(pOperator); return NULL; } + +// ==================================================================================================================== +// TableCountScanOperator +static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); +static void destoryTableCountScanOperator(void* param); + +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pScanNode, + SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + + STableCountScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + + if (!pInfo || !pOperator) { + goto _error; + } + + pInfo->readHandle = *readHandle; + + if (pScanNode->pScanPseudoCols != NULL) { + SExprSupp* pSup = &pInfo->pseudoSup; + pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + } + + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + initResultSizeInfo(&pOperator->resultInfo, 1); + pInfo->pRes = createResDataBlock(pDescNode); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + tNameAssign(&pInfo->name, &pScanNode->tableName); + + pInfo->suid = pScanNode->suid; // 0 for super table, super table uid for child table + pInfo->uid = pScanNode->uid; // super table uid, or child table uid + pInfo->tableType = pScanNode->tableType; + + setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, NULL); + return pOperator; + +_error: + if (pInfo != NULL) { + destoryTableCountScanOperator(pInfo); + } + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableCountScanInfo* pInfo = pOperator->info; + + SExprSupp* pSup = &pInfo->pseudoSup; + if (pSup->numOfExprs != 1 || pSup->pExprInfo[0].pExpr->nodeType != QUERY_NODE_FUNCTION || + pSup->pExprInfo[0].pExpr->_function.functionType != FUNCTION_TYPE_TABLE_COUNT ) { + qError("%s table count scan operator invalid pseduo columns", GET_TASKID(pTaskInfo)); + } + SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); + if (pInfo->uid != 0) { + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, pInfo->uid, &stats); + int64_t ctbNum = stats.ctbNum; + //TODO: wxy about the return type bigint or int? + colDataAppend(pColInfoData, 0, (char*)&ctbNum, false); + pInfo->pRes->info.rows = 1; + } else { + //TODO: get table count in this vnode? + } + return NULL; +} + +static void destoryTableCountScanOperator(void* param) { + STableCountScanInfo* pTableCountScanInfo = param; + blockDataDestroy(pTableCountScanInfo->pRes); + + cleanupExprSupp(&pTableCountScanInfo->pseudoSup); + taosMemoryFreeClear(param); +} From 8d77b0cfb71e39c75134e79cae0293f8661fa141 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 20:42:00 +0800 Subject: [PATCH 006/120] fix: save work --- source/libs/executor/src/scanoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a7298df58d..941f7e7098 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4794,10 +4794,11 @@ _error: static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); static void destoryTableCountScanOperator(void* param); -SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pScanNode, +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pTblCountScanNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; + SScanPhysiNode* pScanNode = &pTblCountScanNode->scan; STableCountScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); From c2a7404b425e655bf01e54e159f79b96e430b047 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 25 Nov 2022 23:00:14 +0800 Subject: [PATCH 007/120] save work in case it losts --- source/dnode/vnode/inc/vnode.h | 2 + source/dnode/vnode/src/inc/vnodeInt.h | 2 - source/libs/executor/inc/executorimpl.h | 36 ++-- source/libs/executor/src/scanoperator.c | 227 ++++++++++++++++++++---- 4 files changed, 216 insertions(+), 51 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 52a8a7f5a3..791e7e88b0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -108,6 +108,8 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); +int64_t metaGetTbNum(SMeta* pMeta); typedef struct { int64_t uid; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1ca80c7570..1b15e55c0d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -116,8 +116,6 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); -tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); -int64_t metaGetTbNum(SMeta* pMeta); int64_t metaGetTimeSeriesNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock); void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1785553c39..a14c9bb51e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -530,16 +530,32 @@ typedef struct SSysTableScanInfo { SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; -typedef struct STableCountScanInfo { - SReadHandle readHandle; - SSDataBlock* pRes; - SExprSupp pseudoSup; - SNode* pCondition; - SName name; - uint64_t suid; - uint64_t uid; - int8_t tableType; -} STableCountScanInfo; +typedef struct STableCountScanSupp { + int16_t dbNameSlotId; + int16_t stbNameSlotId; + int16_t tbCountSlotId; + + bool groupByDbName; + bool groupByStbName; + char dbName[TSDB_DB_NAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + +} STableCountScanSupp; + +typedef struct STableCountScanOperatorInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + + SName tableName; + + SNodeList* groupTags; + SNodeList* scanCols; + SNodeList* pseudoCols; + + STableCountScanSupp supp; + + int32_t currGrpIdx; +} STableCountScanOperatorInfo; typedef struct SBlockDistInfo { SSDataBlock* pResBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 941f7e7098..c8ff156cd3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -31,7 +31,6 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -233,8 +232,8 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro STableScanInfo* pTableScanInfo = pOperator->info; - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->base.pdInfo.pAggSup->pResultRowHashTable, buf, - GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->base.pdInfo.pAggSup->pResultRowHashTable, + buf, GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); if (p1 == NULL) { return NULL; @@ -376,7 +375,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; @@ -4428,7 +4427,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { uint32_t status = 0; loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); -// code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); + // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -4758,7 +4757,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN goto _error; } - initResultSizeInfo(&pOperator->resultInfo, 1024); pInfo->pResBlock = createResDataBlock(pDescNode); blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -4792,15 +4790,106 @@ _error: // ==================================================================================================================== // TableCountScanOperator static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); -static void destoryTableCountScanOperator(void* param); +static void destoryTableCountScanOperator(void* param); +static const char* GROUP_TAG_DB_NAME = "db_name"; +static const char* GROUP_TAG_STABLE_NAME = "stable_name"; + +int32_t tblCountScanGetGroupTagsSlotId(const SNodeList* scanCols, STableCountScanSupp* supp) { + if (scanCols != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, scanCols) { + if (nodeType(pNode) != QUERY_NODE_TARGET) { + return TSDB_CODE_QRY_SYS_ERROR; + } + STargetNode* targetNode = (STargetNode*)pNode; + if (nodeType(targetNode->pExpr) != QUERY_NODE_COLUMN) { + return TSDB_CODE_QRY_SYS_ERROR; + } + SColumnNode* colNode = (SColumnNode*)(targetNode->pExpr); + if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) { + supp->dbNameSlotId = targetNode->slotId; + } else if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) { + supp->stbNameSlotId = targetNode->slotId; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t tblCountScanGetCountSlotId(const SNodeList* pseudoCols, STableCountScanSupp* supp) { + if (pseudoCols != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, pseudoCols) { + if (nodeType(pNode) != QUERY_NODE_TARGET) { + return TSDB_CODE_QRY_SYS_ERROR; + } + STargetNode* targetNode = (STargetNode*)pNode; + if (nodeType(targetNode->pExpr) != QUERY_NODE_FUNCTION) { + return TSDB_CODE_QRY_SYS_ERROR; + } + SFunctionNode* funcNode = (SFunctionNode*)(targetNode->pExpr); + if (funcNode->funcType == FUNCTION_TYPE_TABLE_COUNT) { + supp->tbCountSlotId = targetNode->slotId; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCountScanSupp* supp) { + if (groupTags != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, groupTags) { + if (nodeType(pNode) != QUERY_NODE_COLUMN) { + return TSDB_CODE_QRY_SYS_ERROR; + } + SColumnNode* colNode = (SColumnNode*)pNode; + if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) { + supp->groupByDbName = true; + } + if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) { + supp->groupByStbName = true; + } + } + } else { + strncpy(supp->dbName, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); + strncpy(supp->stbName, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); + } + return TSDB_CODE_SUCCESS; +} + +int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* scanCols, SNodeList* pseudoCols, + STableCountScanSupp* supp, SExecTaskInfo* taskInfo) { + int32_t code = 0; + code = tblCountScanGetInputs(groupTags, tableName, supp); + if (code != TSDB_CODE_SUCCESS) { + qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo)); + return code; + } + supp->dbNameSlotId = -1; + supp->stbNameSlotId = -1; + supp->tbCountSlotId = -1; + + code = tblCountScanGetGroupTagsSlotId(scanCols, supp); + if (code != TSDB_CODE_SUCCESS) { + qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo)); + return code; + } + code = tblCountScanGetCountSlotId(pseudoCols, supp); + if (code != TSDB_CODE_SUCCESS) { + qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo)); + return code; + } + return code; +} SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pTblCountScanNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; - SScanPhysiNode* pScanNode = &pTblCountScanNode->scan; - STableCountScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SScanPhysiNode* pScanNode = &pTblCountScanNode->scan; + STableCountScanOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanSupp)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (!pInfo || !pOperator) { goto _error; @@ -4808,26 +4897,19 @@ SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableC pInfo->readHandle = *readHandle; - if (pScanNode->pScanPseudoCols != NULL) { - SExprSupp* pSup = &pInfo->pseudoSup; - pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); - } - SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; initResultSizeInfo(&pOperator->resultInfo, 1); pInfo->pRes = createResDataBlock(pDescNode); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - tNameAssign(&pInfo->name, &pScanNode->tableName); - - pInfo->suid = pScanNode->suid; // 0 for super table, super table uid for child table - pInfo->uid = pScanNode->uid; // super table uid, or child table uid - pInfo->tableType = pScanNode->tableType; + getTableCountScanSupp(pTblCountScanNode->pGroupTags, &pTblCountScanNode->scan.tableName, + pTblCountScanNode->scan.pScanCols, pTblCountScanNode->scan.pScanPseudoCols, &pInfo->supp, + pTaskInfo); setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, NULL); return pOperator; _error: @@ -4840,32 +4922,99 @@ _error: } static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableCountScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableCountScanOperatorInfo* pInfo = pOperator->info; + STableCountScanSupp* pSupp = &pInfo->supp; + SSDataBlock* pRes = pInfo->pRes; + // compute group id must, but output is according to scancols. output group by group + // grouptags high priority(groupid<=>grouptag), then tablename(dbname,tableName). + // scanCols, (grouptags cols) + { + // mnode, query table count of information_schema and performance_schema + if (pInfo->readHandle.mnd != NULL) { + if (pSupp->groupByDbName) { + if (pInfo->currGrpIdx == 0) { + uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + size_t infodbTableNum; + getInfosDbMeta(NULL, &infodbTableNum); + pRes->info.groupId = groupId; + if (pSupp->dbNameSlotId != -1) { + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); + //colDataAppend(colInfoData, 0, ) + } + return NULL; + } else if (pInfo->currGrpIdx == 1) { + uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + size_t perfdbTableNum; + getPerfDbMeta(NULL, &perfdbTableNum); + } else if (pInfo->currGrpIdx >= 2) { + return NULL; + } + uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + size_t infodbTableNum; + getInfosDbMeta(NULL, &infodbTableNum); + size_t perfdbTableNum; + getPerfDbMeta(NULL, &perfdbTableNum); + // grouptags, db name + } else { + } + } + return NULL; + } + const char* db = NULL; + int32_t vgId = 0; + { + // get dbname + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); + SName sn = {0}; + char varDbName[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - SExprSupp* pSup = &pInfo->pseudoSup; - if (pSup->numOfExprs != 1 || pSup->pExprInfo[0].pExpr->nodeType != QUERY_NODE_FUNCTION || - pSup->pExprInfo[0].pExpr->_function.functionType != FUNCTION_TYPE_TABLE_COUNT ) { - qError("%s table count scan operator invalid pseduo columns", GET_TASKID(pTaskInfo)); + tNameGetDbName(&sn, varDataVal(varDbName)); + varDataSetLen(varDbName, strlen(varDataVal(varDbName))); } - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); - if (pInfo->uid != 0) { + const char* tableName = tNameGetTableName(&pInfo->tableName); + + { + // grouptags only have column db_name or (no grouptags and tablename is null) + // if (tableName == NULL | strlen(tableName) == 0) + metaGetTbNum(pInfo->readHandle.meta); + } + {// no grouptags and TableName is not null, return child table count + {tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, tableName); + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + int64_t ctbNum = stats.ctbNum; +} +} +{ + // grouptags have column stable_name. return (stable name, child table count) + SArray* stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); + vnodeGetStbIdList(pInfo->readHandle.vnode, 0, stbUidList); + if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, stbUidList) < 0) { + qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); + taosArrayDestroy(stbUidList); + // return failure + } + for (int i = 0; i < taosArrayGetSize(stbUidList); ++i) { + tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(stbUidList, i); SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, pInfo->uid, &stats); + metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); int64_t ctbNum = stats.ctbNum; - //TODO: wxy about the return type bigint or int? - colDataAppend(pColInfoData, 0, (char*)&ctbNum, false); - pInfo->pRes->info.rows = 1; - } else { - //TODO: get table count in this vnode? + + char varStbName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + metaGetTableNameByUid(pInfo->readHandle.meta, stbUid, varStbName); } - return NULL; + taosArrayDestroy(stbUidList); +} +return NULL; } static void destoryTableCountScanOperator(void* param) { - STableCountScanInfo* pTableCountScanInfo = param; + STableCountScanOperatorInfo* pTableCountScanInfo = param; blockDataDestroy(pTableCountScanInfo->pRes); - cleanupExprSupp(&pTableCountScanInfo->pseudoSup); + nodesDestroyList(pTableCountScanInfo->groupTags); taosMemoryFreeClear(param); } From 2cccfe2be44b695309b188502bec944503f888ae Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 26 Nov 2022 23:03:12 +0800 Subject: [PATCH 008/120] feature: save work --- source/libs/executor/src/scanoperator.c | 122 +++++++++++++++++------- 1 file changed, 86 insertions(+), 36 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c8ff156cd3..51d317ea0f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4921,68 +4921,118 @@ _error: return NULL; } +void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* stbName, int64_t count, + SSDataBlock* pRes) { + if (pSupp->dbNameSlotId != -1) { + ASSERT(strlen(dbName)); + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); + char varDbName[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + strncpy(varDataVal(varDbName), dbName, strlen(dbName)); + varDataSetLen(varDbName, strlen(dbName)); + colDataAppend(colInfoData, 0, varDbName, false); + } + + if (pSupp->stbNameSlotId != -1) { + ASSERT(strlen(stbName)); + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); + char varStbName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + strncpy(varDataVal(varStbName), stbName, strlen(stbName)); + varDataSetLen(varStbName, strlen(stbName)); + colDataAppend(colInfoData, 0, varStbName, false); + } + + if (pSupp->tbCountSlotId != -1) { + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); + colDataAppend(colInfoData, 0, (char*)&count, false); + } + pRes->info.rows = 1; +} + +static SSDataBlock* buildSysDbTableCount(STableCountScanOperatorInfo* pInfo) { + STableCountScanSupp* pSupp = &pInfo->supp; + SSDataBlock* pRes = pInfo->pRes; + + int64_t infodbTableNum; + getInfosDbMeta(NULL, &infodbTableNum); + int64_t perfdbTableNum; + getPerfDbMeta(NULL, &perfdbTableNum); + + if (pSupp->groupByDbName) { + if (pInfo->currGrpIdx == 0) { + uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + pRes->info.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (pInfo->currGrpIdx == 1) { + uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + pRes->info.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } + pInfo->currGrpIdx++; + return (pRes->info.rows > 0) ? pRes : NULL; + } else { + if (strcmp(pSupp->dbName, TSDB_INFORMATION_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (strcmp(pSupp->dbName, TSDB_PERFORMANCE_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else if (strlen(pSupp->dbName) == 0) { + fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); + } + return (pRes->info.rows > 0) ? pRes : NULL; + } +} + static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableCountScanOperatorInfo* pInfo = pOperator->info; STableCountScanSupp* pSupp = &pInfo->supp; SSDataBlock* pRes = pInfo->pRes; + blockDataCleanup(pRes); // compute group id must, but output is according to scancols. output group by group // grouptags high priority(groupid<=>grouptag), then tablename(dbname,tableName). // scanCols, (grouptags cols) - { - // mnode, query table count of information_schema and performance_schema - if (pInfo->readHandle.mnd != NULL) { - if (pSupp->groupByDbName) { - if (pInfo->currGrpIdx == 0) { - uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); - size_t infodbTableNum; - getInfosDbMeta(NULL, &infodbTableNum); - pRes->info.groupId = groupId; - if (pSupp->dbNameSlotId != -1) { - SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); - //colDataAppend(colInfoData, 0, ) - } - return NULL; - } else if (pInfo->currGrpIdx == 1) { - uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); - size_t perfdbTableNum; - getPerfDbMeta(NULL, &perfdbTableNum); - } else if (pInfo->currGrpIdx >= 2) { - return NULL; - } - uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); - size_t infodbTableNum; - getInfosDbMeta(NULL, &infodbTableNum); - size_t perfdbTableNum; - getPerfDbMeta(NULL, &perfdbTableNum); - // grouptags, db name - } else { - } - } - return NULL; + // mnode, query table count of information_schema and performance_schema + if (pInfo->readHandle.mnd != NULL) { + return buildSysDbTableCount(pInfo); } + const char* db = NULL; int32_t vgId = 0; + char dbName[TSDB_DB_FNAME_LEN] = {0}; + { // get dbname vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); SName sn = {0}; - char varDbName[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&sn, varDataVal(varDbName)); - varDataSetLen(varDbName, strlen(varDataVal(varDbName))); + tNameGetDbName(&sn, dbName); } - const char* tableName = tNameGetTableName(&pInfo->tableName); + if (pSupp->groupByDbName) { + if (pSupp->groupByStbName) { + } else { + uint64_t groupId = calcGroupId(dbName, strlen(dbName)); + pRes->info.groupId = groupId; + int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); + } + } else { + if (strlen(pSupp->dbName) != 0) { + if (strlen(pSupp->stbName) != 0) { + + } else { + + } + } + } { // grouptags only have column db_name or (no grouptags and tablename is null) // if (tableName == NULL | strlen(tableName) == 0) metaGetTbNum(pInfo->readHandle.meta); } {// no grouptags and TableName is not null, return child table count - {tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, tableName); + {tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbName); SMetaStbStats stats = {0}; metaGetStbStats(pInfo->readHandle.meta, uid, &stats); int64_t ctbNum = stats.ctbNum; From b5b6fff68b2b77d80f6deaec7594de5a5c44f046 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 27 Nov 2022 13:50:36 +0800 Subject: [PATCH 009/120] feature: save work --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/meta/metaQuery.c | 16 +++++ source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/scanoperator.c | 79 +++++++++++++------------ 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 791e7e88b0..3fcd9fbc2b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -105,6 +105,7 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 620022c06d..7f8353049a 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -211,6 +211,22 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } + +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { + int code = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, (SMeta *)meta, 0); + code = metaGetTableEntryByUid(&mr, uid); + if (code < 0) { + metaReaderClear(&mr); + return -1; + } + strncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); + metaReaderClear(&mr); + + return 0; +} + int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid) { int code = 0; SMetaReader mr = {0}; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a14c9bb51e..3f1b7605ea 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -555,6 +555,7 @@ typedef struct STableCountScanOperatorInfo { STableCountScanSupp supp; int32_t currGrpIdx; + SArray* stbUidList; // when group by db_name and stable_name } STableCountScanOperatorInfo; typedef struct SBlockDistInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 51d317ea0f..78a8f9e706 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4926,7 +4926,7 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* if (pSupp->dbNameSlotId != -1) { ASSERT(strlen(dbName)); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); - char varDbName[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varDbName), dbName, strlen(dbName)); varDataSetLen(varDbName, strlen(dbName)); colDataAppend(colInfoData, 0, varDbName, false); @@ -4935,7 +4935,7 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* if (pSupp->stbNameSlotId != -1) { ASSERT(strlen(stbName)); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); - char varStbName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varStbName), stbName, strlen(stbName)); varDataSetLen(varStbName, strlen(stbName)); colDataAppend(colInfoData, 0, varStbName, false); @@ -4997,7 +4997,7 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - char dbName[TSDB_DB_FNAME_LEN] = {0}; + char dbName[TSDB_DB_NAME_LEN] = {0}; { // get dbname @@ -5005,13 +5005,40 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&sn, dbName); } if (pSupp->groupByDbName) { if (pSupp->groupByStbName) { + // group by db_name and stable_name + if (pInfo->stbUidList == NULL) { + pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); + if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); + } + } + if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { + tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); + int64_t ctbNum = stats.ctbNum; + + char stbName[TSDB_TABLE_NAME_LEN] = {0}; + metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + strcpy(fullStbName, dbName); + strcat(fullStbName, "."); + strcat(fullStbName, stbName); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.groupId = groupId; + + pInfo->currGrpIdx++; + + fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); + } } else { + // group by only db_name uint64_t groupId = calcGroupId(dbName, strlen(dbName)); pRes->info.groupId = groupId; int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); @@ -5020,45 +5047,18 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { } else { if (strlen(pSupp->dbName) != 0) { if (strlen(pSupp->stbName) != 0) { - + tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbName); + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + int64_t ctbNum = stats.ctbNum; + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbName, ctbNum, pRes); } else { - + int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, pSupp->dbName, "", tbNumVnode, pRes); } } } - { - // grouptags only have column db_name or (no grouptags and tablename is null) - // if (tableName == NULL | strlen(tableName) == 0) - metaGetTbNum(pInfo->readHandle.meta); - } - {// no grouptags and TableName is not null, return child table count - {tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbName); - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, uid, &stats); - int64_t ctbNum = stats.ctbNum; -} -} -{ - // grouptags have column stable_name. return (stable name, child table count) - SArray* stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - vnodeGetStbIdList(pInfo->readHandle.vnode, 0, stbUidList); - if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, stbUidList) < 0) { - qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); - taosArrayDestroy(stbUidList); - // return failure - } - for (int i = 0; i < taosArrayGetSize(stbUidList); ++i) { - tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(stbUidList, i); - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - - char varStbName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - metaGetTableNameByUid(pInfo->readHandle.meta, stbUid, varStbName); - } - taosArrayDestroy(stbUidList); -} -return NULL; + return pRes->info.rows > 0 ? pRes : pRes; } static void destoryTableCountScanOperator(void* param) { @@ -5066,5 +5066,6 @@ static void destoryTableCountScanOperator(void* param) { blockDataDestroy(pTableCountScanInfo->pRes); nodesDestroyList(pTableCountScanInfo->groupTags); + taosArrayDestroy(pTableCountScanInfo->stbUidList); taosMemoryFreeClear(param); } From b289adce9b7349a1a160f94dfd1f850e3ced7ef8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 28 Nov 2022 11:12:06 +0800 Subject: [PATCH 010/120] fix: save work --- source/libs/executor/src/scanoperator.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 78a8f9e706..743b5d8a84 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -5002,7 +5002,6 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { { // get dbname vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); @@ -5019,23 +5018,21 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - char stbName[TSDB_TABLE_NAME_LEN] = {0}; metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - strcpy(fullStbName, dbName); - strcat(fullStbName, "."); - strcat(fullStbName, stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.groupId = groupId; - pInfo->currGrpIdx++; + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); + int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); + + pInfo->currGrpIdx++; } } else { // group by only db_name From 797af5373ddca04177a64da4c78ebcab517e3b7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 10:59:40 +0800 Subject: [PATCH 011/120] enh(query): improve the multi-way merge performance. --- source/libs/executor/src/exchangeoperator.c | 7 +--- source/libs/executor/src/sortoperator.c | 5 ++- source/libs/executor/src/tsort.c | 46 +++++++++++---------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index b2ddff45a4..001d668749 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -209,7 +209,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { } } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { +static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -318,7 +318,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); return pOperator; _error: @@ -581,13 +581,10 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - tsem_post(&pExchangeInfo->ready); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ec754f31b0..2f504cee3d 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -559,7 +559,7 @@ typedef struct SMultiwayMergeOperatorInfo { STupleHandle* prefetchedTuple; } SMultiwayMergeOperatorInfo; -int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { +int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { SMultiwayMergeOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -580,6 +580,7 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; ps->onlyRef = true; + tsortAddSource(pInfo->pSortHandle, ps); } @@ -792,7 +793,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, + pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 3f91142708..4db76b82f5 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -34,14 +34,12 @@ struct SSortHandle { int32_t pageSize; int32_t numOfPages; SDiskbasedBuf* pBuf; - - SArray* pSortInfo; - SArray* pOrderedSource; - - int32_t loops; - uint64_t sortElapsed; - int64_t startTs; - uint64_t totalElapsed; + SArray* pSortInfo; + SArray* pOrderedSource; + int32_t loops; + uint64_t sortElapsed; + int64_t startTs; + uint64_t totalElapsed; int32_t sourceId; SSDataBlock* pDataBlock; @@ -99,9 +97,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page } static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { + // NOTICE: pSource may be, if it is SORT_MULTISOURCE_MERGE for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = - cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE + SSortSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); taosMemoryFreeClear(pSource); } @@ -228,15 +226,15 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList); } -static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) { +static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) { pSource->src.rowIndex = -1; ++pHandle->numOfCompletedSources; } -static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, +static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { - cmpParam->pSources = taosArrayGet(pSources, startIndex); - cmpParam->numOfSources = (endIndex - startIndex + 1); + pParam->pSources = taosArrayGet(pSources, startIndex); + pParam->numOfSources = (endIndex - startIndex + 1); int32_t code = 0; @@ -244,7 +242,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { code = TSDB_CODE_NO_AVAIL_DISK; - qError("Sort compare init failed since %s", terrstr(code)); + qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr); return code; } @@ -257,12 +255,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } if (pHandle->type == SORT_SINGLESOURCE_SORT) { - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = cmpParam->pSources[i]; + for (int32_t i = 0; i < pParam->numOfSources; ++i) { + SSortSource* pSource = pParam->pSources[i]; // set current source is done if (taosArrayGetSize(pSource->pageIdList) == 0) { - setCurrentSourceIsDone(pSource, pHandle); + setCurrentSourceDone(pSource, pHandle); continue; } @@ -277,15 +275,21 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int releaseBufPage(pHandle->pBuf, pPage); } } else { - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = cmpParam->pSources[i]; + qDebug("start init for the multiway merge sort, %s", pHandle->idStr); + int64_t st = taosGetTimestampUs(); + + for (int32_t i = 0; i < pParam->numOfSources; ++i) { + SSortSource* pSource = pParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); // set current source is done if (pSource->src.pBlock == NULL) { - setCurrentSourceIsDone(pSource, pHandle); + setCurrentSourceDone(pSource, pHandle); } } + + int64_t et = taosGetTimestampUs(); + qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); } return code; From a1585f16f10076cffc0d2c82737b4e42e5c3fedb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 11:06:20 +0800 Subject: [PATCH 012/120] ehn(query): improve the merge sort performance. --- source/libs/executor/src/sortoperator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 2f504cee3d..0287af62d2 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -577,8 +577,13 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SOperatorInfo* pDownstream = pOperator->pDownstream[i]; + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { + pDownstream->fpSet._openFn(pDownstream); + } + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = pOperator->pDownstream[i]; + ps->param = pDownstream; ps->onlyRef = true; tsortAddSource(pInfo->pSortHandle, ps); From dac930352b7764c3f0b36b9703a8251ad1d31784 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 12:22:04 +0800 Subject: [PATCH 013/120] enh(query): optimize the perf in compare. --- include/common/ttypes.h | 2 - source/libs/scalar/src/sclvector.c | 62 +++++++++++++++++++----------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 761ffd0f1c..6350057c1f 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -278,11 +278,9 @@ typedef struct { #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c4ff5b2b01..98391d95f4 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1543,9 +1543,44 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t step, __compar_fn_t fp, int32_t optr) { int32_t num = 0; - bool * pRes = (bool *)pOut->columnData->pData; + bool *pRes = (bool *)pOut->columnData->pData; - if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) { + if (IS_MATHABLE_TYPE(GET_PARAM_TYPE(pLeft)) && IS_MATHABLE_TYPE(GET_PARAM_TYPE(pRight))) { + if (!(pLeft->columnData->hasNull || pRight->columnData->hasNull)) { + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; + int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; + + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); + + pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + if (pRes[i]) { + ++num; + } + } + } else { + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; + int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; + + if (colDataIsNull_f(pLeft->columnData->nullbitmap, leftIndex) || + colDataIsNull_f(pRight->columnData->nullbitmap, rightIndex)) { + pRes[i] = false; + continue; + } + + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); + + pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + if (pRes[i]) { + ++num; + } + } + } + } else { + // if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) { for (int32_t i = startIndex; i < numOfRows && i >= startIndex; i += step) { int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; @@ -1556,8 +1591,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa continue; } - char * pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char * pRightData = colDataGetData(pRight->columnData, rightIndex); + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); int64_t leftOut = 0; int64_t rightOut = 0; bool freeLeft = false; @@ -1592,25 +1627,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa taosMemoryFreeClear(pRightData); } } - } else { - for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { - int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; - int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; - - if (colDataIsNull_s(pLeft->columnData, leftIndex) || - colDataIsNull_s(pRight->columnData, rightIndex)) { - pRes[i] = false; - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char *pRightData = colDataGetData(pRight->columnData, rightIndex); - - pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); - if (pRes[i]) { - ++num; - } - } } return num; From 37b65c16c8e32cabb991ff1a8637399a62aa3f23 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 29 Nov 2022 14:59:48 +0800 Subject: [PATCH 014/120] enh: ins_tables count optimize --- source/libs/function/src/builtins.c | 2 +- source/libs/planner/src/planOptimizer.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 3e2bc2a4ac..a2a827e7e5 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2081,7 +2081,7 @@ static int32_t translateTagsPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in } static int32_t translateTableCountPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 515d594e45..4f288f97d3 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2630,9 +2630,11 @@ static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { pInfo->pScan->scanType = SCAN_TYPE_TABLE_COUNT; strcpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname); strcpy(pInfo->pScan->tableName.tname, pInfo->table.tname); - NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); + if (NULL == pInfo->pAgg->pGroupKeys) { + NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); + NODES_DESTORY_LIST(pInfo->pScan->pScanCols); + } NODES_DESTORY_NODE(pInfo->pScan->node.pConditions); - NODES_DESTORY_LIST(pInfo->pScan->pScanCols); NODES_DESTORY_LIST(pInfo->pScan->pScanPseudoCols); int32_t code = nodesListMakeStrictAppend(&pInfo->pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc()); if (TSDB_CODE_SUCCESS == code) { @@ -2672,7 +2674,7 @@ static int32_t tbCntScanOptRewriteAgg(SAggLogicNode* pAgg) { SNode* pSum = NULL; int32_t code = tbCntScanOptCreateSumFunc( (SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0), - nodesListGetNode(((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->pTargets, 0), &pSum); + nodesListGetNode(((SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->pScanPseudoCols, 0), &pSum); if (TSDB_CODE_SUCCESS == code) { NODES_DESTORY_LIST(pAgg->pAggFuncs); code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, pSum); From 8ee170bef1c5b5a64c6d415d0e8317c74e31034e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 29 Nov 2022 16:17:05 +0800 Subject: [PATCH 015/120] enh: add retry for vnode closed case --- include/libs/executor/executor.h | 2 +- include/util/taoserror.h | 1 + source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executor.c | 4 ++-- source/libs/executor/src/executorimpl.c | 15 ++------------- source/libs/qworker/inc/qwInt.h | 2 +- source/libs/qworker/src/qwUtil.c | 4 ++-- source/libs/qworker/src/qworker.c | 18 ++++++++++-------- source/libs/qworker/test/qworkerTests.cpp | 2 +- source/util/src/terror.c | 1 + 10 files changed, 22 insertions(+), 29 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 0bca254e14..1413b41756 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -152,7 +152,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); * @param tinfo qhandle * @return */ -int32_t qAsyncKillTask(qTaskInfo_t tinfo); +int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); /** * destroy query info structure diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e5d0bcb249..ba7c72e0f3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -323,6 +323,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0526) #define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527) #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) +#define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8163217039..e18a6916dd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -796,7 +796,7 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t checkForQueryBuf(size_t numOfTables); bool isTaskKilled(SExecTaskInfo* pTaskInfo); -void setTaskKilled(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); void doDestroyTask(SExecTaskInfo* pTaskInfo); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 34bd9cf8ca..fd4dd3bd12 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -688,7 +688,7 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) { taosWUnLockLatch(&pTaskInfo->stopInfo.lock); } -int32_t qAsyncKillTask(qTaskInfo_t qinfo) { +int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; if (pTaskInfo == NULL) { @@ -697,7 +697,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) { qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo); + setTaskKilled(pTaskInfo, rspCode); qStopTaskOperators(pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 300ba52934..2766b48744 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -611,21 +611,10 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } bool isTaskKilled(SExecTaskInfo* pTaskInfo) { - // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived - // abort current query execution. - if (pTaskInfo->owner != 0 && - ((taosGetTimestampSec() - pTaskInfo->cost.start / 1000) > 10 * getMaximumIdleDurationSec()) - /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) { - assert(pTaskInfo->cost.start != 0); - // qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 - // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); - // return true; - } - - return false; + return (0 != pTaskInfo->code) ? true : false; } -void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; } +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } ///////////////////////////////////////////////////////////////////////////////////////////// STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index a0e04b6a19..af361323a7 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -363,7 +363,7 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx); -int32_t qwKillTaskHandle(SQWTaskCtx *ctx); +int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode); int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status); int32_t qwDropTask(QW_FPARAMS_DEF); void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 2c0a4072ae..86fd1d533c 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -279,14 +279,14 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) { } } -int32_t qwKillTaskHandle(SQWTaskCtx *ctx) { +int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode) { int32_t code = 0; // Note: free/kill may in RC qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { qDebug("start to kill task"); - code = qAsyncKillTask(taskHandle); + code = qAsyncKillTask(taskHandle, rspCode); atomic_store_ptr(&ctx->taskHandle, taskHandle); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 0890d10b65..9a318df324 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -411,7 +411,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu // qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + QW_ERR_JRET(ctx->rspCode); } QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC)); @@ -420,7 +420,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu case QW_PHASE_PRE_FETCH: { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_TASK_WLOG("task dropping or already dropped, phase:%s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + QW_ERR_JRET(ctx->rspCode); } if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { @@ -442,7 +442,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu case QW_PHASE_PRE_CQUERY: { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + QW_ERR_JRET(ctx->rspCode); } if (ctx->rspCode) { @@ -456,7 +456,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu // qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + QW_ERR_JRET(ctx->rspCode); } break; @@ -502,7 +502,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + QW_ERR_JRET(ctx->rspCode); } if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { @@ -515,7 +515,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + QW_ERR_JRET(ctx->rspCode); } if (ctx->rspCode) { @@ -861,7 +861,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (QW_QUERY_RUNNING(ctx)) { - QW_ERR_JRET(qwKillTaskHandle(ctx)); + QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED)); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP); } else { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); @@ -869,6 +869,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (!dropped) { + QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_TSC_QUERY_CANCELLED); QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } @@ -1195,8 +1196,9 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { } if (QW_QUERY_RUNNING(ctx)) { - qwKillTaskHandle(ctx); + qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); } else if (!QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8a48977c77..02b341e28c 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -302,7 +302,7 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock **pRes, uint64_t *useconds) { return 0; } -int32_t qwtKillTask(qTaskInfo_t qinfo) { return 0; } +int32_t qwtKillTask(qTaskInfo_t qinfo, int32_t rspCode) { return 0; } void qwtDestroyTask(qTaskInfo_t qHandle) {} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4d889843e8..35f59c4881 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -315,6 +315,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_NOT_EXISTS, "Table column not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subscribed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") From 445c4f28f0d09f3f38813f5c26f016c10350f265 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 23:34:22 +0800 Subject: [PATCH 016/120] enh(query): optimize the perf. --- source/common/src/tdatablock.c | 5 ++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/sortoperator.c | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a0f795a729..20eda52aac 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1139,6 +1139,9 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF void blockDataCleanup(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; + int32_t existedRows = pInfo->rows; + ASSERT(existedRows <= pDataBlock->info.capacity); + pInfo->rows = 0; pInfo->id.uid = 0; pInfo->id.groupId = 0; @@ -1152,7 +1155,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfoDataCleanup(p, pInfo->capacity); + colInfoDataCleanup(p, existedRows); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4ba311212a..660d189451 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3589,7 +3589,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - colDataAppendInt64(pColData, outputRowIndex, &pBlockData->aTSKEY[rowIndex]); + ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0287af62d2..b355b85861 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -720,7 +720,6 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { } qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo)); - SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -787,7 +786,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc); - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 1024); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); pInfo->groupSort = pMergePhyNode->groupSort; From a8703c7fc22d661d1fbd7e196ddeb475b4ffbfd7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 00:11:21 +0800 Subject: [PATCH 017/120] enh(query): optimize the perf. --- source/libs/executor/src/timewindowoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a6a477a9e3..dae2dafe9e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1286,7 +1286,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { return NULL; } - blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); @@ -1737,7 +1736,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 1024); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); From 3f0c93a62308112a1ba2a5093277fb8c342ff509 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 14:53:25 +0800 Subject: [PATCH 018/120] enh(query): optimize query perf. --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 294a4bd3e4..c149e52fed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd } pFD->szPage = szPage; pFD->pgno = 0; - pFD->pBuf = taosMemoryCalloc(1, szPage); + pFD->pBuf = taosMemoryMalloc(szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosCloseFile(&pFD->pFD); From ae816331441c1602d500a5f3d208da76e1fe323b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 14:55:27 +0800 Subject: [PATCH 019/120] enh(query): optimize query perf. --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 20eda52aac..af45db053a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1242,7 +1242,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false); if (code) { return code; } From 8dcbafb78066547b8f8f04c4772e2f7a559f1415 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 15:24:03 +0800 Subject: [PATCH 020/120] enh(query): avoiding unnecessary remove operation. --- source/util/src/tpagedbuf.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index e1a43ace47..b7dbdad053 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -507,7 +507,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { dBufPrintStatis(pBuf); + bool needRemoveFile = false; if (pBuf->pFile != NULL) { + needRemoveFile = true; uDebug( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " "size:%.2f Kb, %s\n", @@ -534,9 +536,14 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { } } - if (taosRemoveFile(pBuf->path) < 0) { - uDebug("WARNING tPage remove file failed. path=%s", pBuf->path); + if (needRemoveFile) { + int32_t ret = taosRemoveFile(pBuf->path); + if (ret != 0) { // print the error and discard this error info + int32_t code = TAOS_SYSTEM_ERROR(errno); + uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code)); + } } + taosMemoryFreeClear(pBuf->path); size_t n = taosArrayGetSize(pBuf->pIdList); From d66b7f3de07d6bb1149b5be41b5aab6ac0fa2e7e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 15:25:19 +0800 Subject: [PATCH 021/120] refactor: do some internal refactor. --- source/util/src/tpagedbuf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index b7dbdad053..0f4c51060b 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -539,8 +539,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (needRemoveFile) { int32_t ret = taosRemoveFile(pBuf->path); if (ret != 0) { // print the error and discard this error info - int32_t code = TAOS_SYSTEM_ERROR(errno); - uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code)); + uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(errno)); } } From 9465d100a2708b303f31f5420f07752fb6c973db Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 30 Nov 2022 15:31:01 +0800 Subject: [PATCH 022/120] enh: ins_table count optimize --- source/libs/parser/src/parAstParser.c | 3 ++- source/libs/planner/src/planOptimizer.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 8c5806ebe1..d4e486db6b 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -165,7 +165,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } - if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) && + if (TSDB_CODE_SUCCESS == code && + (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) && QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { code = collectMetaKeyFromInsTags(pCxt); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4f288f97d3..6627dfbae8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2658,7 +2658,7 @@ static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, } strcpy(pFunc->functionName, "sum"); strcpy(pFunc->node.aliasName, pCntFunc->node.aliasName); - int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pParam)); + int32_t code = createColumnByRewriteExpr(pParam, &pFunc->pParameterList); if (TSDB_CODE_SUCCESS == code) { code = fmGetFuncInfo(pFunc, NULL, 0); } From 259b893f0a9e31671b1c803ab5c005f223de00c2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 30 Nov 2022 15:42:45 +0800 Subject: [PATCH 023/120] fix: integration test of table count scan --- source/libs/executor/src/scanoperator.c | 31 ++++++++++++++++++------- source/libs/nodes/src/nodesMsgFuncs.c | 4 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index fcdd4dd86b..f01781fd2b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3028,7 +3028,7 @@ SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableC int32_t code = TSDB_CODE_SUCCESS; SScanPhysiNode* pScanNode = &pTblCountScanNode->scan; - STableCountScanOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanSupp)); + STableCountScanOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (!pInfo || !pOperator) { @@ -3082,19 +3082,19 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* } if (pSupp->tbCountSlotId != -1) { - SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->tbCountSlotId); colDataAppend(colInfoData, 0, (char*)&count, false); } pRes->info.rows = 1; } -static SSDataBlock* buildSysDbTableCount(STableCountScanOperatorInfo* pInfo) { +static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo) { STableCountScanSupp* pSupp = &pInfo->supp; SSDataBlock* pRes = pInfo->pRes; - int64_t infodbTableNum; + size_t infodbTableNum; getInfosDbMeta(NULL, &infodbTableNum); - int64_t perfdbTableNum; + size_t perfdbTableNum; getPerfDbMeta(NULL, &perfdbTableNum); if (pSupp->groupByDbName) { @@ -3106,6 +3106,9 @@ static SSDataBlock* buildSysDbTableCount(STableCountScanOperatorInfo* pInfo) { uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); pRes->info.id.groupId = groupId; fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else { + setOperatorCompleted(pOperator); + return NULL; } pInfo->currGrpIdx++; return (pRes->info.rows > 0) ? pRes : NULL; @@ -3117,6 +3120,7 @@ static SSDataBlock* buildSysDbTableCount(STableCountScanOperatorInfo* pInfo) { } else if (strlen(pSupp->dbName) == 0) { fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); } + setOperatorCompleted(pOperator); return (pRes->info.rows > 0) ? pRes : NULL; } } @@ -3131,8 +3135,11 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { // grouptags high priority(groupid<=>grouptag), then tablename(dbname,tableName). // scanCols, (grouptags cols) // mnode, query table count of information_schema and performance_schema + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } if (pInfo->readHandle.mnd != NULL) { - return buildSysDbTableCount(pInfo); + return buildSysDbTableCount(pOperator, pInfo); } const char* db = NULL; @@ -3173,6 +3180,9 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); pInfo->currGrpIdx++; + } else { + setOperatorCompleted(pOperator); + return NULL; } } else { // group by only db_name @@ -3180,6 +3190,7 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { pRes->info.id.groupId = groupId; int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); + setOperatorCompleted(pOperator); } } else { if (strlen(pSupp->dbName) != 0) { @@ -3191,11 +3202,15 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbName, ctbNum, pRes); } else { int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, pSupp->dbName, "", tbNumVnode, pRes); + fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } + } else { + int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } + setOperatorCompleted(pOperator); } - return pRes->info.rows > 0 ? pRes : pRes; + return pRes->info.rows > 0 ? pRes : NULL; } static void destoryTableCountScanOperator(void* param) { diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index d054cda745..bf6cd33af7 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -3637,10 +3637,10 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = physiScanNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = physiLastRowScanNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: @@ -3776,10 +3776,10 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = msgToPhysiScanNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = msgToPhysiLastRowScanNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: From a450fae89a5cacebddd163ee5f3ae95bccf2f207 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 30 Nov 2022 17:00:36 +0800 Subject: [PATCH 024/120] fix:window time wait --- source/os/src/osSemaphore.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index bfce8b3151..3a375732b2 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -76,19 +76,15 @@ int32_t tsem_wait(tsem_t* sem) { } int32_t tsem_timewait(tsem_t* sem, int64_t milis) { - return 0; - /*return tsem_wait(sem);*/ -#if 0 struct timespec ts; - timespec_get(&ts); + taosClockGetTime(0, &ts); + ts.tv_nsec += ms * 1000000; ts.tv_sec += ts.tv_nsec / 1000000000; ts.tv_nsec %= 1000000000; - - /*GetSystemTimeAsFileTime(&ft_before);*/ - // errno = 0; - rc = sem_timedwait(sem, ts); - + int rc; + while ((rc = sem_timedwait(sem, &ts)) == -1 && errno == EINTR) continue; + return rc; /* This should have timed out */ // assert(errno == ETIMEDOUT); // assert(rc != 0); @@ -103,8 +99,6 @@ int32_t tsem_timewait(tsem_t* sem, int64_t milis) { // printf("time must advance during sem_timedwait."); // return 1; // } - return rc; -#endif } #elif defined(_TD_DARWIN_64) From 45ecc2ba529c7c4fc5a4e276f98c0b2220502843 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 30 Nov 2022 19:52:29 +0800 Subject: [PATCH 025/120] fix: no result for table cout scan group by dbname,stable_name for systable --- source/libs/executor/src/scanoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 98b8d46aae..7bcc2e6a0e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3099,6 +3099,10 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc getPerfDbMeta(NULL, &perfdbTableNum); if (pSupp->groupByDbName) { + if (pSupp->groupByStbName) { + setOperatorCompleted(pOperator); + return NULL; + } if (pInfo->currGrpIdx == 0) { uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); pRes->info.id.groupId = groupId; From 4fef284cfe07d71afe4f088b0d7767f79f1ae636 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 21:26:27 +0800 Subject: [PATCH 026/120] refactor: do some internal refactor. --- include/libs/function/function.h | 59 +++++++----------------- include/libs/function/tudf.h | 26 +++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 2 +- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 240772bfc2..8fe9d98e57 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -137,22 +137,22 @@ typedef struct SqlFunctionCtx { int16_t functionId; // function id char *pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; - SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - int32_t offset; - struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaries; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; - struct SExprInfo *pExpr; - struct SSDataBlock *pSrcBlock; - struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity - SSerializeDataHandle saveHandle; - bool isStream; - - char udfName[TSDB_FUNC_NAME_LEN]; + // input parameter, e.g., top(k, 20), the number of results of top query is kept in param + SFunctParam *param; + // corresponding output buffer for timestamp of each result, e.g., diff/csum + SColumnInfoData *pTsOutput; + int32_t offset; + SResultRowEntryInfo *resultInfo; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + struct SExprInfo *pExpr; + struct SSDataBlock *pSrcBlock; + struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity + SSerializeDataHandle saveHandle; + char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; typedef struct tExprNode { @@ -182,7 +182,6 @@ struct SScalarParam { }; void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); -//int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock); bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); @@ -194,32 +193,6 @@ typedef struct SPoint { int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, int32_t inputType); -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// udf api -/** - * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf - * @return error code - */ -int32_t udfcOpen(); - -/** - * destroy udfd proxy - * @return error code - */ -int32_t udfcClose(); - -/** - * start udfd that serves udf function invocation under dnode startDnodeId - * @param startDnodeId - * @return - */ -int32_t udfStartUdfd(int32_t startDnodeId); -/** - * stop udfd - * @return - */ -int32_t udfStopUdfd(); - #ifdef __cplusplus } #endif diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 31cc53bb9f..b71d50d43c 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -85,6 +85,32 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, int32_t cleanUpUdfs(); +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// udf api +/** + * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf + * @return error code + */ +int32_t udfcOpen(); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t udfcClose(); + +/** + * start udfd that serves udf function invocation under dnode startDnodeId + * @param startDnodeId + * @return + */ +int32_t udfStartUdfd(int32_t startDnodeId); +/** + * stop udfd + * @return + */ +int32_t udfStopUdfd(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 8719e988e7..2124b387ec 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -39,7 +39,7 @@ #include "sync.h" #include "wal.h" -#include "libs/function/function.h" +#include "libs/function/tudf.h" #ifdef __cplusplus extern "C" { #endif From 26ab0894a88f97699cbf9cb6a05bd666e0e618b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 21:56:02 +0800 Subject: [PATCH 027/120] fix(query): fix syntax error. --- source/libs/executor/src/executil.c | 2 -- source/libs/executor/src/timewindowoperator.c | 7 +------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b135566caa..7c37d66eb6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1536,8 +1536,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; - pCtx->isStream = false; - pCtx->param = pFunct->pParam; pCtx->saveHandle.currentPage = -1; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index dae2dafe9e..35548a6ade 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1717,7 +1717,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { for (int32_t i = 0; i < numOfExpr; i++) { - pCtx[i].isStream = true; +// pCtx[i].isStream = true; } } @@ -1783,11 +1783,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh goto _error; } - if (isStream) { - ASSERT(num > 0); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - } - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo); if (pInfo->timeWindowInterpo) { From cefe4be1abef27af09104d7a053ecc3478d93bfb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 00:34:06 +0800 Subject: [PATCH 028/120] refactor: do some internal refactor. --- include/common/tdatablock.h | 1 + source/common/src/tdatablock.c | 19 +++++++++ .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 41 ++++++++++--------- source/util/src/tarray.c | 2 +- source/util/src/thash.c | 4 +- 8 files changed, 48 insertions(+), 25 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index db6ca65cf0..a5d3a0829d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -234,6 +234,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index af45db053a..f5e73d8f0e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1239,6 +1239,25 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true); + if (code) { + return code; + } + } + + pDataBlock->info.capacity = numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) { + int32_t code = 0; + if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) { + return TSDB_CODE_SUCCESS; + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c149e52fed..294a4bd3e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd } pFD->szPage = szPage; pFD->pgno = 0; - pFD->pBuf = taosMemoryMalloc(szPage); + pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosCloseFile(&pFD->pFD); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 300ba52934..3e28798a3a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1749,7 +1749,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n pAggSup->currentPageId = -1; pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn); + pAggSup->pResultRowHashTable = tSimpleHashInit(100, hashFn); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84b7678b9f..ca3734d91e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -886,7 +886,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); - blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 35548a6ade..551d668763 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1658,24 +1658,6 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) { // the primary timestamp column bool needed = false; - pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); - - { // ts column - SColumn c = {0}; - c.colId = 1; - c.slotId = pInfo->primaryTsIndex; - c.type = TSDB_DATA_TYPE_TIMESTAMP; - c.bytes = sizeof(int64_t); - taosArrayPush(pInfo->pInterpCols, &c); - - SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = true; // to denote no value is assigned yet - key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(pInfo->pPrevValues, &key); - } for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = pCtx[i].pExpr; @@ -1696,6 +1678,27 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } } + if (needed) { + pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); + + { // ts column + SColumn c = {0}; + c.colId = 1; + c.slotId = pInfo->primaryTsIndex; + c.type = TSDB_DATA_TYPE_TIMESTAMP; + c.bytes = sizeof(int64_t); + taosArrayPush(pInfo->pInterpCols, &c); + + SGroupKeys key; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = true; // to denote no value is assigned yet + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } + } + return needed; } @@ -1737,7 +1740,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 1024); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index f53b1cfd7f..6a16c044f9 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { } pArray->size = 0; - pArray->pData = taosMemoryCalloc(size, elemSize); + pArray->pData = taosMemoryMalloc(size * elemSize); if (pArray->pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pArray); diff --git a/source/util/src/thash.c b/source/util/src/thash.c index a0411483ca..b07d485c87 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -244,7 +244,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp capacity = 4; } - SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj)); + SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -264,7 +264,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *)); if (pHashObj->hashList == NULL) { taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; From c996837fab936338a81491e9c4ae88412091f7ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 01:10:37 +0800 Subject: [PATCH 029/120] refactor: do some internal refactor. --- source/util/src/tarray.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 6a16c044f9..f53b1cfd7f 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { } pArray->size = 0; - pArray->pData = taosMemoryMalloc(size * elemSize); + pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pArray); From 89be771b6172bb04e19b72e8193889daa523bcee Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 1 Dec 2022 09:53:41 +0800 Subject: [PATCH 030/120] enh: add retry for vnode stopped case --- include/libs/qcom/query.h | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f51aa88485..ef543470aa 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -260,7 +260,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) -#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED) #define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 95656fd76c..b9b7477952 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -234,7 +234,8 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcRe static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK || + code == TSDB_CODE_VND_STOPPED) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; From eea8d7bebbcb642c028b03bebb85ec114a71088d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 1 Dec 2022 10:48:36 +0800 Subject: [PATCH 031/120] enh: ins_table count optimize --- source/libs/parser/test/mockCatalog.cpp | 2 +- source/libs/planner/src/planOptimizer.c | 27 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 935ef77b93..bc12861dc7 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -66,8 +66,8 @@ void generateInformationSchema(MockCatalogService* mcs) { .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .done(); mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 3) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .done(); mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 2) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 6627dfbae8..2af09f8f8b 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2630,10 +2630,8 @@ static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { pInfo->pScan->scanType = SCAN_TYPE_TABLE_COUNT; strcpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname); strcpy(pInfo->pScan->tableName.tname, pInfo->table.tname); - if (NULL == pInfo->pAgg->pGroupKeys) { - NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); - NODES_DESTORY_LIST(pInfo->pScan->pScanCols); - } + NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); + NODES_DESTORY_LIST(pInfo->pScan->pScanCols); NODES_DESTORY_NODE(pInfo->pScan->node.pConditions); NODES_DESTORY_LIST(pInfo->pScan->pScanPseudoCols); int32_t code = nodesListMakeStrictAppend(&pInfo->pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc()); @@ -2642,8 +2640,14 @@ static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { } SNode* pGroupKey = NULL; FOREACH(pGroupKey, pInfo->pAgg->pGroupKeys) { - code = nodesListMakeStrictAppend( - &pInfo->pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + SNode* pGroupCol = nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0); + code = nodesListMakeStrictAppend(&pInfo->pScan->pGroupTags, nodesCloneNode(pGroupCol)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pScan->pScanCols, nodesCloneNode(pGroupCol)); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pScan->node.pTargets, nodesCloneNode(pGroupCol)); + } if (TSDB_CODE_SUCCESS != code) { break; } @@ -2671,14 +2675,17 @@ static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, } static int32_t tbCntScanOptRewriteAgg(SAggLogicNode* pAgg) { - SNode* pSum = NULL; - int32_t code = tbCntScanOptCreateSumFunc( - (SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0), - nodesListGetNode(((SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->pScanPseudoCols, 0), &pSum); + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0); + SNode* pSum = NULL; + int32_t code = tbCntScanOptCreateSumFunc((SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0), + nodesListGetNode(pScan->pScanPseudoCols, 0), &pSum); if (TSDB_CODE_SUCCESS == code) { NODES_DESTORY_LIST(pAgg->pAggFuncs); code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, pSum); } + if (TSDB_CODE_SUCCESS == code) { + code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, 0, pAgg); + } NODES_DESTORY_LIST(pAgg->pGroupKeys); return code; } From 58c75dde6ff082a1cbb9070b76a619d8bd3265dc Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 1 Dec 2022 11:38:34 +0800 Subject: [PATCH 032/120] fix: add seperate group for normal table when group by db_name/stbable_name --- source/dnode/vnode/inc/vnode.h | 26 +++++++++--------- source/dnode/vnode/src/meta/metaQuery.c | 4 +++ source/libs/executor/src/scanoperator.c | 35 ++++++++++++++----------- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fe2892a76b..756e23deeb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -107,23 +107,23 @@ int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); -int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); -int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, - bool *acquired); -int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, - int32_t payloadLen, double selectivityRatio); -int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); -tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); -int64_t metaGetTbNum(SMeta* pMeta); - +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); +int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, + bool *acquired); +int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen, double selectivityRatio); +int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int64_t metaGetTbNum(SMeta *pMeta); +int64_t metaGetNtbNum(SMeta *pMeta); typedef struct { int64_t uid; int64_t ctbNum; } SMetaStbStats; -int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); +int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); typedef struct SMetaFltParam { tb_uid_t suid; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 8fb93f2c2e..cfdb4ab8d1 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -756,6 +756,10 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; } +int64_t metaGetNtbNum(SMeta *pMeta) { + return pMeta->pVnode->config.vndStats.numOfNTables; +} + typedef struct { SMeta *pMeta; TBC *pCur; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index db508b43f9..437e3e1393 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3050,7 +3050,7 @@ _error: void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* stbName, int64_t count, SSDataBlock* pRes) { if (pSupp->dbNameSlotId != -1) { - ASSERT(strlen(dbName)); + ASSERT(strlen(pSupp->dbName)); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varDbName), dbName, strlen(dbName)); @@ -3059,12 +3059,15 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* } if (pSupp->stbNameSlotId != -1) { - ASSERT(strlen(stbName)); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); - char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - strncpy(varDataVal(varStbName), stbName, strlen(stbName)); - varDataSetLen(varStbName, strlen(stbName)); - colDataAppend(colInfoData, 0, varStbName, false); + if (strlen(stbName) != 0) { + char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + strncpy(varDataVal(varStbName), stbName, strlen(stbName)); + varDataSetLen(varStbName, strlen(stbName)); + colDataAppend(colInfoData, 0, varStbName, false); + } else { + colDataAppendNULL(colInfoData, 0); + } } if (pSupp->tbCountSlotId != -1) { @@ -3084,10 +3087,6 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc getPerfDbMeta(NULL, &perfdbTableNum); if (pSupp->groupByDbName) { - if (pSupp->groupByStbName) { - setOperatorCompleted(pOperator); - return NULL; - } if (pInfo->currGrpIdx == 0) { uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); pRes->info.id.groupId = groupId; @@ -3121,10 +3120,7 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { STableCountScanSupp* pSupp = &pInfo->supp; SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); - // compute group id must, but output is according to scancols. output group by group - // grouptags high priority(groupid<=>grouptag), then tablename(dbname,tableName). - // scanCols, (grouptags cols) - // mnode, query table count of information_schema and performance_schema + if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -3145,7 +3141,6 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { } if (pSupp->groupByDbName) { if (pSupp->groupByStbName) { - // group by db_name and stable_name if (pInfo->stbUidList == NULL) { pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { @@ -3169,13 +3164,21 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); + pInfo->currGrpIdx++; + } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + pInfo->currGrpIdx++; } else { setOperatorCompleted(pOperator); return NULL; } } else { - // group by only db_name uint64_t groupId = calcGroupId(dbName, strlen(dbName)); pRes->info.id.groupId = groupId; int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); From 6e73c89e504038577d273880637689a14343f500 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 1 Dec 2022 12:22:03 +0800 Subject: [PATCH 033/120] fix: executor returns kill task error code --- source/libs/executor/src/exchangeoperator.c | 6 +++--- source/libs/executor/src/scanoperator.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 963a273290..53660d88e1 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -70,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + longjmp(pTaskInfo->env, pTaskInfo->code); } for (int32_t i = 0; i < totalSources; ++i) { @@ -573,7 +573,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + longjmp(pTaskInfo->env, pTaskInfo->code); } tsem_post(&pExchangeInfo->ready); @@ -621,7 +621,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + longjmp(pTaskInfo->env, pTaskInfo->code); } SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b6353061fb..3c4ed00000 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -629,7 +629,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) { if (isTaskKilled(pTaskInfo)) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } // process this data block based on the probabilities @@ -2032,7 +2032,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + longjmp(pTaskInfo->env, pTaskInfo->code); } int32_t rows = 0; @@ -2529,7 +2529,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { STsdbReader* reader = pInfo->base.dataReader; while (tsdbNextDataBlock(reader)) { if (isTaskKilled(pTaskInfo)) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } // process this data block based on the probabilities From 98726f17fa13b21fe88fd41f2536396528833f16 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 1 Dec 2022 14:54:58 +0800 Subject: [PATCH 034/120] fix: redirect handler issue --- source/libs/scheduler/src/schTask.c | 10 +++++----- source/libs/transport/src/transCli.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9a7f3332b3..289fb35032 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -430,15 +430,15 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 if (SCH_IS_DATA_BIND_TASK(pTask)) { if (pData && pData->pEpSet) { SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); - } else if (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(rspCode)) { - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SCH_SWITCH_EPSET(addr); - SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); - } else { + } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port); + } else { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SCH_SWITCH_EPSET(addr); + SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index fbcc1fb525..4003a74ec7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1512,7 +1512,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || TSDB_CODE_VND_STOPPED) { tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 3378ff0fb89d8e8fd2b698e865e2fba3c6f02cf9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Dec 2022 16:03:40 +0800 Subject: [PATCH 035/120] refactor code --- source/libs/transport/inc/transComm.h | 4 +-- source/libs/transport/src/transCli.c | 44 +++++++++++++++++++-------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 479c1a5af7..bf9a6c0051 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -151,8 +151,8 @@ typedef struct { int64_t retryNextInterval; bool retryInit; int32_t retryStep; - - int8_t epsetRetryCnt; + int8_t epsetRetryCnt; + int32_t retryCode; int hThrdIdx; } STransConnCtx; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index fbcc1fb525..e92b44f8c2 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1020,7 +1020,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tDebug("current epset %s", tbuf); if (!EPSET_IS_VALID(&pCtx->epSet)) { tError("invalid epset"); @@ -1500,34 +1499,46 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryNextInterval = pCtx->retryMinInterval; pCtx->retryStep = 0; pCtx->retryInit = true; + pCtx->retryCode = TSDB_CODE_SUCCESS; } + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { return false; } + // code, msgType + + // A: epset, leader, not self + // B: epset, not know leader + // C: no epset, leader but not serivce + bool noDelay = false; if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT) { - tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); + tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); addConnToPool(pThrd->pool, pConn); } else if (code == TSDB_CODE_SYN_RESTORING) { - tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); addConnToPool(pThrd->pool, pConn); transFreeMsg(pResp->pCont); } else { - tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); addConnToPool(pThrd->pool, pConn); transFreeMsg(pResp->pCont); } + if (code != TSDB_CODE_RPC_BROKEN_LINK && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && code != TSDB_CODE_SUCCESS) { + // save one internal code + pCtx->retryCode = code; + } if (noDelay == false) { pCtx->epsetRetryCnt = 1; @@ -1556,29 +1567,36 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STrans* pTransInst = pThrd->pTransInst; if (pMsg == NULL || pMsg->ctx == NULL) { - tDebug("%s conn %p handle resp", pTransInst->label, pConn); + tTrace("%s conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } STransConnCtx* pCtx = pMsg->ctx; - int32_t code = pResp->code; bool retry = cliGenRetryRule(pConn, pResp, pMsg); if (retry == true) { return -1; } - STraceId* trace = &pResp->info.traceId; - bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); + if (pCtx->retryCode != TSDB_CODE_SUCCESS) { + int32_t code = pResp->code; + // return internal code app + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { + pResp->code = pCtx->retryCode; + } + } + + STraceId* trace = &pResp->info.traceId; + bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); if (hasEpSet) { char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGDebug("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); + tGTrace("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); } if (pCtx->pSem != NULL) { - tGDebug("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); + tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pCtx->pRsp == NULL) { tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); } else { @@ -1587,11 +1605,11 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tsem_post(pCtx->pSem); pCtx->pRsp = NULL; } else { - tGDebug("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); + tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (retry == false && hasEpSet == true) { pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); } else { - if (!cliIsEpsetUpdated(code, pCtx)) { + if (!cliIsEpsetUpdated(pResp->code, pCtx)) { pTransInst->cfp(pTransInst->parent, pResp, NULL); } else { pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); From b8ed0459a605094a5ab2d29c1b0429bc6b582c7a Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 1 Dec 2022 16:50:23 +0800 Subject: [PATCH 036/120] fix: change the paramter instead of table name --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 437e3e1393..3dd027ad04 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3050,7 +3050,7 @@ _error: void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* stbName, int64_t count, SSDataBlock* pRes) { if (pSupp->dbNameSlotId != -1) { - ASSERT(strlen(pSupp->dbName)); + ASSERT(strlen(dbName)); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varDbName), dbName, strlen(dbName)); From a2de81e9085b9abf2d0137a8bfda18ae5e40648a Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 1 Dec 2022 16:58:47 +0800 Subject: [PATCH 037/120] fix: fill scan type for select count(*) from ins_tables --- source/libs/executor/src/executorimpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 60749aa552..b539d63037 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1381,7 +1381,8 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan int32_t type = pOperator->operatorType; if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN) { + type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { *order = TSDB_ORDER_ASC; *scanFlag = MAIN_SCAN; return TSDB_CODE_SUCCESS; From f8a2ab8338cedbdb68d3f2c1fc4236c1d95fee57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 17:24:26 +0800 Subject: [PATCH 038/120] refactor: do some internal refactor. --- include/common/tcommon.h | 5 +- include/common/tdatablock.h | 6 +- source/common/src/tdatablock.c | 34 +++-- source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 123 ++++++++---------- source/libs/executor/src/executil.c | 13 +- source/libs/executor/src/executor.c | 4 +- source/libs/executor/src/executorimpl.c | 2 - source/libs/executor/src/scanoperator.c | 39 ++---- source/libs/executor/src/sysscanoperator.c | 85 ++++++------ source/libs/executor/src/timewindowoperator.c | 6 +- source/libs/scalar/src/sclvector.c | 2 +- source/util/src/thash.c | 5 +- 13 files changed, 156 insertions(+), 171 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 77f1879b81..6ec3d5db10 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -249,10 +249,11 @@ typedef struct SColumnInfoData { typedef struct SQueryTableDataCond { uint64_t suid; - int32_t order; // desc|asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo* colList; - int32_t type; // data block load type: + int32_t* pSlotList; // the column output destation slot, and it may be null + int32_t type; // data block load type: STimeWindow twindows; int64_t startVersion; int64_t endVersion; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a5d3a0829d..4208d85648 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,7 +41,7 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -#define colDataSetNotNull_f(bm_, r_) \ +#define colDataClearNull_f(bm_, r_) \ do { \ BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ } while (0) @@ -151,9 +151,6 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_f(pColumnInfoData->nullbitmap, i); } - - int32_t bytes = pColumnInfoData->info.bytes; - memset(pColumnInfoData->pData + start * bytes, 0, nRows * bytes); } pColumnInfoData->hasNull = true; @@ -238,6 +235,7 @@ int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRo void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); +void blockDataEmpty(SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f5e73d8f0e..c300c3e6f0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1137,17 +1137,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF } void blockDataCleanup(SSDataBlock* pDataBlock) { + blockDataEmpty(pDataBlock); SDataBlockInfo* pInfo = &pDataBlock->info; - - int32_t existedRows = pInfo->rows; - ASSERT(existedRows <= pDataBlock->info.capacity); - - pInfo->rows = 0; pInfo->id.uid = 0; pInfo->id.groupId = 0; - pInfo->window.ekey = 0; - pInfo->window.skey = 0; +} +void blockDataEmpty(SSDataBlock* pDataBlock) { + SDataBlockInfo* pInfo = &pDataBlock->info; + ASSERT(pInfo->rows <= pDataBlock->info.capacity); if (pInfo->capacity == 0) { return; } @@ -1155,8 +1153,12 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfoDataCleanup(p, existedRows); + colInfoDataCleanup(p, pInfo->capacity); } + + pInfo->rows = 0; + pInfo->window.ekey = 0; + pInfo->window.skey = 0; } // todo temporarily disable it @@ -1643,6 +1645,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + + // clear the offset value of the unused entries. memset(&pColInfoData->varmeta.offset[total - n], 0, n); } else { int32_t bytes = pColInfoData->info.bytes; @@ -1657,7 +1661,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { } if (pBlock->info.rows <= n) { - blockDataCleanup(pBlock); + blockDataEmpty(pBlock); } else { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -1674,12 +1678,22 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); + } else { // reset the bitmap value + int32_t stopIndex = BitmapLen(n) * 8; + for(int32_t i = n + 1; i < stopIndex; ++i) { + colDataClearNull_f(pColInfoData->nullbitmap, i); + } + + int32_t remain = BitmapLen(total) - BitmapLen(n); + if (remain > 0) { + memset(pColInfoData->nullbitmap, 0, remain); + } } } int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { - blockDataCleanup(pBlock); + blockDataEmpty(pBlock); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7ef3207b4d..53b8eadb33 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -159,11 +159,10 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - STsdbReader **ppReader, const char *idstr); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); -bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 660d189451..27d99bd42d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -85,7 +85,8 @@ typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; SColumnDataAgg** plist; - int16_t* colIds; // column ids for loading file block data + int16_t* colIds; // column ids for loading file block data + int16_t* slotIds; // the ordinal index in the destination SSDataBlock int32_t numOfCols; char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. bool smaValid; // the sma on all queried columns are activated @@ -214,25 +215,25 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SSDataBlock* pBlock) { - size_t numOfCols = blockDataGetNumOfCols(pBlock); - +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) { + pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); + if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) { taosMemoryFree(pSupInfo->colIds); taosMemoryFree(pSupInfo->buildBuf); + taosMemoryFree(pSupInfo->slotIds); return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); - pSupInfo->colIds[i] = pCol->info.colId; + pSupInfo->colIds[i] = pCols[i].colId; + pSupInfo->slotIds[i] = pSlotIdList[i]; - if (IS_VAR_DATA_TYPE(pCol->info.type)) { - pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); + if (IS_VAR_DATA_TYPE(pCols[i].type)) { + pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); } } @@ -566,7 +567,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, - const char* idstr) { + SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; int8_t level = 0; STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); @@ -585,6 +586,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->suid = pCond->suid; pReader->order = pCond->order; pReader->capacity = capacity; + pReader->pResBlock = pResBlock; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; @@ -592,6 +594,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket ASSERT(pCond->numOfCols > 0); + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); // allocate buffer in order to load data blocks from file @@ -611,13 +614,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - pReader->pResBlock = createResBlock(pCond, pReader->capacity); - if (pReader->pResBlock == NULL) { - code = terrno; - goto _end; - } - - setColumnIdSlotList(&pReader->suppInfo, pReader->pResBlock); + setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols); *ppReader = pReader; return code; @@ -1041,17 +1038,16 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { - SReaderStatus* pStatus = &pReader->status; - SDataBlockIter* pBlockIter = &pStatus->blockIter; + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pStatus->fileBlockData; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; - int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); - - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int32_t numOfOutputCols = pSupInfo->numOfCols; SColVal cv = {0}; int64_t st = taosGetTimestampUs(); @@ -1087,8 +1083,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); - if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1097,12 +1093,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t num = pBlockData->nColData; while (i < numOfOutputCols && colIndex < num) { rowIndex = 0; - pColData = taosArrayGet(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pColData->info.colId) { + if (pData->cid < pSupInfo->colIds[i]) { colIndex += 1; - } else if (pData->cid == pColData->info.colId) { + } else if (pData->cid == pSupInfo->colIds[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); } else { @@ -1119,6 +1116,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1126,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, i); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -3535,7 +3533,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) { - int32_t numOfRows = pBlock->info.rows; + int32_t outputRowIndex = pBlock->info.rows; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); int64_t uid = pScanInfo->uid; @@ -3545,23 +3543,26 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* SColVal colVal = {0}; int32_t i = 0, j = 0; - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - if (pColInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - colDataAppend(pColInfoData, numOfRows, (const char*)&pTSRow->ts, false); + if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } while (i < numOfCols && j < pSchema->numOfCols) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - col_id_t colId = pColInfoData->info.colId; + col_id_t colId = pSupInfo->colIds[i]; if (colId == pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + tTSRowGetVal(pTSRow, pSchema, j, &colVal); - doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo); + doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - colDataAppendNULL(pColInfoData, numOfRows); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + + colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } else if (colId > pSchema->columns[j].colId) { j += 1; @@ -3570,8 +3571,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* // set null value since current column does not exist in the "pSchema" while (i < numOfCols) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - colDataAppendNULL(pColInfoData, numOfRows); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3586,27 +3587,25 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); - if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } SColVal cv = {0}; int32_t numOfInputCols = pBlockData->nColData; - int32_t numOfOutputCols = pResBlock->pDataBlock->size; + int32_t numOfOutputCols = pSupInfo->numOfCols; while (i < numOfOutputCols && j < numOfInputCols) { - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i); - SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - - if (pData->cid < pCol->info.colId) { + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pSupInfo->colIds[i]) { j += 1; continue; } - if (pData->cid == pCol->info.colId) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pData->cid == pSupInfo->colIds[i]) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3619,7 +3618,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNULL(pCol, outputRowIndex); i += 1; } @@ -3718,14 +3717,14 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - STsdbReader** ppReader, const char* idstr) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { STimeWindow window = pCond->twindows; if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { pCond->twindows.skey += 1; pCond->twindows.ekey -= 1; } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr); + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3751,7 +3750,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } // here we only need one more row, so the capacity is set to be ONE. - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr); + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3765,7 +3764,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } pCond->order = order; - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr); + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3837,10 +3836,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; -_err: + _err: tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; -} + } void tsdbReaderClose(STsdbReader* pReader) { if (pReader == NULL) { @@ -3874,7 +3873,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pSupInfo->colIds); taosArrayDestroy(pSupInfo->pColAgg); - for (int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) { + for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { if (pSupInfo->buildBuf[i] != NULL) { taosMemoryFreeClear(pSupInfo->buildBuf[i]); } @@ -3891,8 +3890,6 @@ void tsdbReaderClose(STsdbReader* pReader) { clearBlockScanInfoBuf(&pReader->blockInfoBuf); } - blockDataDestroy(pReader->pResBlock); - if (pReader->pFileReader != NULL) { tsdbDataFReaderClose(&pReader->pFileReader); } @@ -4007,16 +4004,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } -bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { - STableBlockScanInfo* pBlockScanInfo = - *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); - if (pBlockScanInfo == NULL) { // no data block for the table of given uid - return false; - } - - return true; -} - static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { ASSERT(pReader != NULL); *rows = pReader->pResBlock->info.rows; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7c37d66eb6..3f62191ca1 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1241,6 +1241,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod } } + // set the output flag for each column in SColMatchInfo, according to the *numOfOutputCols = 0; int32_t num = LIST_LENGTH(pOutputNodeList->pSlots); for (int32_t i = 0; i < num; ++i) { @@ -1599,20 +1600,22 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); + pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); return terrno; } - // pCond->twindow = pTableScanNode->scanRange; // TODO: get it from stable scan node pCond->twindows = pTableScanNode->scanRange; pCond->suid = pTableScanNode->scan.suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { @@ -1625,6 +1628,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->colList[j].type = pColNode->node.resType.type; pCond->colList[j].bytes = pColNode->node.resType.bytes; pCond->colList[j].colId = pColNode->colId; + + pCond->pSlotList[j] = pNode->slotId; j += 1; } @@ -1962,7 +1967,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, - struct SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo) { int64_t st = taosGetTimestampUs(); const char* idStr = GET_TASKID(pTaskInfo); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 34bd9cf8ca..d675ac05d5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1064,7 +1064,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - &pTableScanInfo->base.dataReader, NULL) < 0 || + pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || pTableScanInfo->base.dataReader == NULL) { ASSERT(0); } @@ -1116,7 +1116,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); ASSERT(size == 1); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3e28798a3a..aa2682c490 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1190,8 +1190,6 @@ void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { // T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); -// // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ca3734d91e..1729c9f1e8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -285,7 +285,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) { if (pLimitInfo->remainOffset >= pBlock->info.rows) { pLimitInfo->remainOffset -= pBlock->info.rows; - pBlock->info.rows = 0; + blockDataEmpty(pBlock); qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id); } else { blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); @@ -390,7 +390,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca return terrno; } - relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value @@ -638,16 +637,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - blockDataCleanup(pBlock); - SDataBlockInfo* pBInfo = &pBlock->info; - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window); - - blockDataEnsureCapacity(pBlock, rows); // todo remove it latter - pBInfo->rows = rows; - - ASSERT(pBInfo->id.uid != 0); + ASSERT(pBlock->info.id.uid != 0); pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); uint32_t status = 0; @@ -778,7 +768,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + pInfo->pResBlock, (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -879,14 +869,14 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->base.readHandle = *readHandle; + pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; + pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; - initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); - blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -993,10 +983,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; SSDataBlock* pBlock = pTableScanInfo->pResBlock; - blockDataCleanup(pBlock); - STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, + int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -2301,7 +2289,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; - code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL); + code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, &pTSInfo->base.dataReader, NULL); if (code != 0) { terrno = code; destroyTableScanOperatorInfo(pTableScanOp); @@ -2531,11 +2519,10 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { blockDataCleanup(pBlock); int64_t st = taosGetTimestampUs(); - - void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; - int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2553,12 +2540,6 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } blockDataCleanup(pBlock); - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - if (pQueryCond->order == TSDB_ORDER_ASC) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; } else { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 3d35326749..7ac6cee2af 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1899,53 +1899,52 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo) { - SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } + SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } - { - SQueryTableDataCond cond = {0}; + pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pResBlock, 1); - int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); - - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); - cleanupQueryTableDataCond(&cond); - if (code != 0) { - goto _error; - } - } - - pInfo->readHandle = *readHandle; - pInfo->uid = pBlockScanNode->suid; - - pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pResBlock, 1); - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + { + SQueryTableDataCond cond = {0}; + int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); if (code != TSDB_CODE_SUCCESS) { - goto _error; + goto _error; } - setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, - OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); - return pOperator; + STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); - _error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str); + cleanupQueryTableDataCond(&cond); + if (code != 0) { + goto _error; + } + } + + pInfo->readHandle = *readHandle; + pInfo->uid = pBlockScanNode->suid; + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + return pOperator; + +_error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } \ No newline at end of file diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 551d668763..5f077c98d7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1739,8 +1739,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 1024); - blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + initResultSizeInfo(&pOperator->resultInfo, 512); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); @@ -4319,7 +4319,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 512); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 98391d95f4..6074c72424 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1782,7 +1782,7 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, if (colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; colDataAppendInt8(pOut->columnData, i, &v); - colDataSetNotNull_f(pOut->columnData->nullbitmap, i); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } } pOut->columnData->hasNull = false; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index b07d485c87..e9548613aa 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -279,7 +279,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return NULL; } - void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry)); + void *p = taosMemoryMalloc(pHashObj->capacity * sizeof(SHashEntry)); if (p == NULL) { taosArrayDestroy(pHashObj->pMemBlock); taosMemoryFree(pHashObj->hashList); @@ -290,6 +290,9 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp for (int32_t i = 0; i < pHashObj->capacity; ++i) { pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); + pHashObj->hashList[i]->num = 0; + pHashObj->hashList[i]->latch = 0; + pHashObj->hashList[i]->next = NULL; } taosArrayPush(pHashObj->pMemBlock, &p); From 9f55ca7f80fa3c8ff83fbc471c74353264fa82e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 17:45:33 +0800 Subject: [PATCH 039/120] test:update the test cases. --- tests/script/tsim/parser/groupby-basic.sim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index 4c70399e4b..2e436945e0 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -78,8 +78,8 @@ $ts2 = $tb2 . .ts print ===============================groupby_operation print -print ==== select count(*), c1 from group_tb0 group by c1 -sql select count(*), c1 from group_tb0 group by c1 +print ==== select count(*), c1 from group_tb0 group by c1 order by c1 +sql select count(*), c1 from group_tb0 group by c1 order by c1 print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 @@ -98,18 +98,18 @@ endi if $data90 != 10 then return -1 endi -if $data01 != 7 then +if $data01 != 0 then return -1 endi -if $data11 != 6 then +if $data11 != 1 then return -1 endi -if $data91 != 3 then +if $data91 != 9 then return -1 endi -print ==== select first(ts),c1 from group_tb0 group by c1; -sql select first(ts),c1 from group_tb0 group by c1; +print ==== select first(ts),c1 from group_tb0 group by c1 order by c1; +sql select first(ts),c1 from group_tb0 group by c1 order by c1; print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 @@ -120,16 +120,16 @@ if $row != 10 then return -1 endi -if $data00 != @22-01-01 00:00:00.007@ then +if $data00 != @22-01-01 00:00:00.000@ then return -1 endi -if $data01 != 7 then +if $data01 != 0 then return -1 endi -if $data90 != @22-01-01 00:00:00.003@ then +if $data90 != @22-01-01 00:00:00.009@ then return -1 endi -if $data91 != 3 then +if $data91 != 9 then return -1 endi From 233377444c990a5d21920bb777ce98f51d9f67fa Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 1 Dec 2022 17:57:53 +0800 Subject: [PATCH 040/120] enh: ins_table count optimize --- source/libs/command/inc/commandInt.h | 1 + source/libs/command/src/explain.c | 48 +++++++++++++++++++++++++ source/libs/parser/src/parTranslater.c | 2 +- source/libs/planner/src/planOptimizer.c | 3 ++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 4d0c5389e1..6acf19218d 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -34,6 +34,7 @@ extern "C" { #define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s" #define EXPLAIN_DISTBLK_SCAN_FORMAT "Block Dist Scan on %s" #define EXPLAIN_LASTROW_SCAN_FORMAT "Last Row Scan on %s" +#define EXPLAIN_TABLE_COUNT_SCAN_FORMAT "Table Count Row Scan on %s" #define EXPLAIN_PROJECTION_FORMAT "Projection" #define EXPLAIN_JOIN_FORMAT "%s" #define EXPLAIN_AGG_FORMAT "Aggragate" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 410e62a18b..03c7249294 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -19,6 +19,7 @@ #include "query.h" #include "tcommon.h" #include "tdatablock.h" +#include "systable.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); @@ -212,6 +213,11 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = lastRowPhysiNode->scan.node.pChildren; break; } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { + STableCountScanPhysiNode *tableCountPhysiNode = (STableCountScanPhysiNode *)pNode; + pPhysiChildren = tableCountPhysiNode->scan.node.pChildren; + break; + } case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SGroupSortPhysiNode *groupSortPhysiNode = (SGroupSortPhysiNode *)pNode; pPhysiChildren = groupSortPhysiNode->node.pChildren; @@ -1355,6 +1361,48 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { + STableCountScanPhysiNode *pLastRowNode = (STableCountScanPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_TABLE_COUNT_SCAN_FORMAT, + ('\0' != pLastRowNode->scan.tableName.tname[0] ? pLastRowNode->scan.tableName.tname : TSDB_INS_TABLE_TABLES)); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, LIST_LENGTH(pLastRowNode->scan.pScanCols)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + if (pLastRowNode->scan.pScanPseudoCols) { + EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pLastRowNode->scan.pScanPseudoCols->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pLastRowNode->scan.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pLastRowNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pLastRowNode->scan.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_APPEND_LIMIT(pLastRowNode->scan.node.pLimit); + EXPLAIN_ROW_APPEND_SLIMIT(pLastRowNode->scan.node.pSlimit); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pLastRowNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pLastRowNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SGroupSortPhysiNode *pSortNode = (SGroupSortPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_GROUP_SORT_FORMAT); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4c361b91d3..becf58b401 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2257,7 +2257,7 @@ static int32_t getVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, if (TSDB_CODE_SUCCESS == code) { code = getVnodeSysTableVgroupListImpl(pCxt, &targetName, pName, pVgs); } - *pHasUserDbCond = (0 != targetName.type); + *pHasUserDbCond = (0 != targetName.type && taosArrayGetSize(*pVgs) > 0); return code; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 2af09f8f8b..c370e2c84b 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2598,6 +2598,9 @@ static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) { 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags) { return false; } + if (1 == pInfo->pScan->pVgroupList->numOfVgroups && MNODE_HANDLE == pInfo->pScan->pVgroupList->vgroups[0].vgId) { + return false; + } return tbCntScanOptIsEligibleConds(pInfo, pInfo->pScan->node.pConditions); } From 7ad0226e32968d5c467091bec6e9d167826197ff Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 1 Dec 2022 18:04:13 +0800 Subject: [PATCH 041/120] fix: add debug code info --- source/libs/scheduler/src/schTask.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 289fb35032..c8b8db367f 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -391,6 +391,8 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) int64_t leftTime = tsMaxRetryWaitTime - lastTime; pTask->delayExecMs = leftTime < pCtx->periodMs ? leftTime : pCtx->periodMs; + pCtx->roundTimes = 0; + goto _return; } @@ -407,7 +409,7 @@ _return: int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; - SCH_TASK_DLOG("task will be redirected now, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode)); if (NULL == pData) { pTask->retryTimes = 0; @@ -433,8 +435,8 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; - SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, - addr->epSet.numOfEps, pEp->fqdn, pEp->port); + SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse, + addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode)); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SCH_SWITCH_EPSET(addr); From 8c62d5f9fa027c28382ec84641fdd37101cc38d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 18:34:26 +0800 Subject: [PATCH 042/120] fix(query): fix bug in tsdbread. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 +++-- source/libs/executor/src/scanoperator.c | 13 ++++---- source/libs/executor/src/timewindowoperator.c | 33 +++++++++++-------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 27d99bd42d..7c39e378cb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1621,7 +1621,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, 0); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2493,7 +2493,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, 0); + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -3534,7 +3534,6 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); int64_t uid = pScanInfo->uid; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -3549,7 +3548,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* i += 1; } - while (i < numOfCols && j < pSchema->numOfCols) { + while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { col_id_t colId = pSupInfo->colIds[i]; if (colId == pSchema->columns[j].colId) { @@ -3570,7 +3569,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* } // set null value since current column does not exist in the "pSchema" - while (i < numOfCols) { + while (i < pSupInfo->numOfCols) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1729c9f1e8..9fc3575491 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2513,10 +2513,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t readIdx = source->readerIdx; SSDataBlock* pBlock = source->inputBlock; - STableMergeScanInfo* pTableScanInfo = pOperator->info; - SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx); - blockDataCleanup(pBlock); + SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); int64_t st = taosGetTimestampUs(); void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); @@ -2534,12 +2532,11 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } // process this data block based on the probabilities - bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample); + bool processThisBlock = processBlockWithProbability(&pInfo->sample); if (!processThisBlock) { continue; } - blockDataCleanup(pBlock); if (pQueryCond->order == TSDB_ORDER_ASC) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; } else { @@ -2547,7 +2544,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } uint32_t status = 0; - loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); + loadDataBlock(pOperator, &pInfo->base, pBlock, &status); // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -2561,7 +2558,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; - pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; tsdbReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; @@ -2641,6 +2638,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { param.readerIdx = i; param.pOperator = pOperator; param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); + blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity); + taosArrayPush(pInfo->sortSourceParams, ¶m); SQueryTableDataCond cond; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5f077c98d7..52f06ffab4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1659,22 +1659,11 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt // the primary timestamp column bool needed = false; - for (int32_t i = 0; i < numOfCols; ++i) { + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = pCtx[i].pExpr; - if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) { - SFunctParam* pParam = &pExpr->base.pParam[0]; - - SColumn c = *pParam->pCol; - taosArrayPush(pInfo->pInterpCols, &c); needed = true; - - SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(pInfo->pPrevValues, &key); + break; } } @@ -1699,6 +1688,24 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } } + for (int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = pCtx[i].pExpr; + + if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) { + SFunctParam* pParam = &pExpr->base.pParam[0]; + + SColumn c = *pParam->pCol; + taosArrayPush(pInfo->pInterpCols, &c); + + SGroupKeys key = {0}; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } + } + return needed; } From 740daefdc54a49a0451566a28db77b6cee9098f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 18:39:15 +0800 Subject: [PATCH 043/120] fix(query): fix a typo. --- source/libs/executor/src/scanoperator.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9fc3575491..87aacfdd34 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -996,15 +996,13 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU if (hasBlock) { SDataBlockInfo* pBInfo = &pBlock->info; - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); +// int32_t rows = 0; +// tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); - doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows); +// relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid); } From 7facf64e5a01b3e7ac55f7c581d964e638f88c9a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 1 Dec 2022 18:58:30 +0800 Subject: [PATCH 044/120] fix: optimize ctg code --- source/libs/catalog/inc/catalogInt.h | 2 + source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgCache.c | 59 ++++++++++-- source/libs/catalog/src/ctgUtil.c | 138 +++++++++++++++++++++++---- 4 files changed, 173 insertions(+), 28 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c95d0fe462..d73dcb3b30 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -764,6 +764,8 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); +int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); + int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d1551..f17abaa31f 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -536,7 +536,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* return TSDB_CODE_SUCCESS; } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue2(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 19b7ee32ae..db3a53aeb0 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -277,6 +277,49 @@ _return: return TSDB_CODE_SUCCESS; } + +int32_t ctgAcquireStbMetaFromCache2(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { + SCtgTbCache *pCache = NULL; + char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + taosHashRelease(dbCache->stbCache, stName); + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pTb = pCache; + + ctgDebug("stb 0x%" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(numOfMetaHit, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(numOfMetaMiss, 1); + + *pTb = NULL; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgAcquireTbIndexFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; @@ -384,17 +427,17 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** memcpy(*pTableMeta, tbMeta, metaSize); - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + } + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); - if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - taosMemoryFreeClear(*pTableMeta); - ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); - return TSDB_CODE_SUCCESS; - } + ctgAcquireStbMetaFromCache2(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f795be3ac2..ef7b80a676 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -837,6 +837,57 @@ _return: CTG_RET(code); } +int ctgVgInfoComp2(const void* lp, const void* rp) { + SVgroupInfo* pLeft = (SVgroupInfo*)lp; + SVgroupInfo* pRight = (SVgroupInfo*)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + +int32_t ctgHashValueComp(void const* lp, void const* rp) { + uint32_t* key = (uint32_t*)lp; + SVgroupInfo* pVg = *(SVgroupInfo**)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + +int ctgVgInfoComp(const void* lp, const void* rp) { + SVgroupInfo* pLeft = *(SVgroupInfo**)lp; + SVgroupInfo* pRight = *(SVgroupInfo**)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + + +int32_t ctgHashValueComp2(void const* lp, void const* rp) { + uint32_t* key = (uint32_t*)lp; + SVgroupInfo* pVg = (SVgroupInfo*)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; @@ -869,7 +920,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + ctgError("0no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -883,29 +934,78 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName CTG_RET(code); } -int32_t ctgHashValueComp(void const* lp, void const* rp) { - uint32_t* key = (uint32_t*)lp; - SVgroupInfo* pVg = *(SVgroupInfo**)rp; - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; +int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { + int32_t code = 0; + + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); + + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); } - return 0; -} + SVgroupInfo* vgInfo = NULL; + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); -int ctgVgInfoComp(const void* lp, const void* rp) { - SVgroupInfo* pLeft = *(SVgroupInfo**)lp; - SVgroupInfo* pRight = *(SVgroupInfo**)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, + dbInfo->hashPrefix, dbInfo->hashSuffix); + + static SArray* pVgList = NULL; + static bool created = false; + + if (!created) { + ctgDebug("create vg array, %d", taosHashGetSize(dbInfo->vgHash)); + pVgList = taosArrayInit(100, sizeof(SVgroupInfo)); + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(pVgList, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(pVgList, ctgVgInfoComp2); + created = true; } - return 0; + vgInfo = taosArraySearch(pVgList, &hashValue, ctgHashValueComp2, TD_EQ); + + if (NULL == vgInfo) { + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (vgInfo) { + taosArrayDestroy(pVgList); + created = false; + ctgDebug("need to re-create vg array, %d", taosHashGetSize(dbInfo->vgHash)); + } + } + + if (NULL == vgInfo) { + ctgError("1no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *pVgroup = *vgInfo; + + ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, + vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, + vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + CTG_RET(code); } int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, @@ -982,7 +1082,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* SVgroupInfo** p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); if (NULL == p) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + ctgError("2no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); From ee5d0a384f8df9f342e984a3be9de6c504aa2038 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 1 Dec 2022 19:05:33 +0800 Subject: [PATCH 045/120] fix: refactor table scan count --- source/libs/executor/inc/executorimpl.h | 12 +- source/libs/executor/src/scanoperator.c | 228 ++++++++++++++---------- 2 files changed, 141 insertions(+), 99 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3896a8a419..9d797f8520 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -489,8 +489,8 @@ typedef struct STableCountScanSupp { bool groupByDbName; bool groupByStbName; - char dbName[TSDB_DB_NAME_LEN]; - char stbName[TSDB_TABLE_NAME_LEN]; + char dbNameFilter[TSDB_DB_NAME_LEN]; + char stbNameFilter[TSDB_TABLE_NAME_LEN]; } STableCountScanSupp; @@ -498,16 +498,10 @@ typedef struct STableCountScanOperatorInfo { SReadHandle readHandle; SSDataBlock* pRes; - SName tableName; - - SNodeList* groupTags; - SNodeList* scanCols; - SNodeList* pseudoCols; - STableCountScanSupp supp; int32_t currGrpIdx; - SArray* stbUidList; // when group by db_name and stable_name + SArray* stbUidList; // when group by db_name and/or stable_name } STableCountScanOperatorInfo; typedef struct SOptrBasicInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3dd027ad04..07ef30f8c5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -486,10 +486,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, - tstrerror(terrno), idStr); + qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", + pBlock->info.id.uid, tstrerror(terrno), idStr); } else { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), + idStr); } metaReaderClear(&mr); return terrno; @@ -1435,8 +1436,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock NULL); if (closedWin && pInfo->partitionSup.needCalc) { gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId); - appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId, - NULL); + appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, + &gpId, NULL); } } } @@ -2461,7 +2462,8 @@ static void destroyTagScanOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2917,6 +2919,21 @@ _error: // TableCountScanOperator static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); static void destoryTableCountScanOperator(void* param); +static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid); +static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName); +static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); +static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); +static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes); +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum); +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum); static const char* GROUP_TAG_DB_NAME = "db_name"; static const char* GROUP_TAG_STABLE_NAME = "stable_name"; @@ -2978,8 +2995,8 @@ int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCoun } } } else { - strncpy(supp->dbName, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); - strncpy(supp->stbName, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); + strncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); + strncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); } return TSDB_CODE_SUCCESS; } @@ -3061,7 +3078,7 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* if (pSupp->stbNameSlotId != -1) { SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); if (strlen(stbName) != 0) { - char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varStbName), stbName, strlen(stbName)); varDataSetLen(varStbName, strlen(stbName)); colDataAppend(colInfoData, 0, varStbName, false); @@ -3087,33 +3104,43 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc getPerfDbMeta(NULL, &perfdbTableNum); if (pSupp->groupByDbName) { - if (pInfo->currGrpIdx == 0) { - uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); - pRes->info.id.groupId = groupId; - fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); - } else if (pInfo->currGrpIdx == 1) { - uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); - pRes->info.id.groupId = groupId; - fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); - } else { - setOperatorCompleted(pOperator); - return NULL; - } - pInfo->currGrpIdx++; + buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum); return (pRes->info.rows > 0) ? pRes : NULL; } else { - if (strcmp(pSupp->dbName, TSDB_INFORMATION_SCHEMA_DB) == 0) { - fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); - } else if (strcmp(pSupp->dbName, TSDB_PERFORMANCE_SCHEMA_DB) == 0) { - fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); - } else if (strlen(pSupp->dbName) == 0) { - fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); - } - setOperatorCompleted(pOperator); + buildSysDbFilterTableCount(pOperator, pSupp, pRes, infodbTableNum, perfdbTableNum); return (pRes->info.rows > 0) ? pRes : NULL; } } +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum) { + if (strcmp(pSupp->dbNameFilter, TSDB_INFORMATION_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (strcmp(pSupp->dbNameFilter, TSDB_PERFORMANCE_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else if (strlen(pSupp->dbNameFilter) == 0) { + fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); + } + setOperatorCompleted(pOperator); +} + +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum) { + if (pInfo->currGrpIdx == 0) { + uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + pRes->info.id.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (pInfo->currGrpIdx == 1) { + uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + pRes->info.id.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else { + setOperatorCompleted(pOperator); + } + pInfo->currGrpIdx++; +} + static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableCountScanOperatorInfo* pInfo = pOperator->info; @@ -3128,89 +3155,110 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { return buildSysDbTableCount(pOperator, pInfo); } + return buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes); +} + +static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes) { const char* db = NULL; int32_t vgId = 0; char dbName[TSDB_DB_NAME_LEN] = {0}; - { - // get dbname - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - SName sn = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&sn, dbName); - } + // get dbname + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + SName sn = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&sn, dbName); + if (pSupp->groupByDbName) { - if (pSupp->groupByStbName) { - if (pInfo->stbUidList == NULL) { - pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { - qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); - } + buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName); + } else { + buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName); + } + return pRes->info.rows > 0 ? pRes : NULL; +} + +static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + if (pSupp->groupByStbName) { + if (pInfo->stbUidList == NULL) { + pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); + if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } - if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { - tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); + } + if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { + tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); + buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid); - char stbName[TSDB_TABLE_NAME_LEN] = {0}; - metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + pInfo->currGrpIdx++; + } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { + buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName); - char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); - uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); - pRes->info.id.groupId = groupId; - - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - - fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); - - pInfo->currGrpIdx++; - } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { - char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); - uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); - pRes->info.id.groupId = groupId; - int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); - - pInfo->currGrpIdx++; - } else { - setOperatorCompleted(pOperator); - return NULL; - } + pInfo->currGrpIdx++; } else { - uint64_t groupId = calcGroupId(dbName, strlen(dbName)); - pRes->info.id.groupId = groupId; - int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } } else { - if (strlen(pSupp->dbName) != 0) { - if (strlen(pSupp->stbName) != 0) { - tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbName); - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, uid, &stats); - int64_t ctbNum = stats.ctbNum; - fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbName, ctbNum, pRes); - } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); - } + uint64_t groupId = calcGroupId(dbName, strlen(dbName)); + pRes->info.id.groupId = groupId; + int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); + setOperatorCompleted(pOperator); + } +} + +static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + if (strlen(pSupp->dbNameFilter) != 0) { + if (strlen(pSupp->stbNameFilter) != 0) { + tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + int64_t ctbNum = stats.ctbNum; + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); } else { int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } - setOperatorCompleted(pOperator); + } else { + int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } - return pRes->info.rows > 0 ? pRes : NULL; + setOperatorCompleted(pOperator); +} + +static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName) { + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); +} + +static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { + char stbName[TSDB_TABLE_NAME_LEN] = {0}; + metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); + int64_t ctbNum = stats.ctbNum; + + fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); } static void destoryTableCountScanOperator(void* param) { STableCountScanOperatorInfo* pTableCountScanInfo = param; blockDataDestroy(pTableCountScanInfo->pRes); - nodesDestroyList(pTableCountScanInfo->groupTags); taosArrayDestroy(pTableCountScanInfo->stbUidList); taosMemoryFreeClear(param); } From c0c1e0daddfa33b230acee003c613f8ba0c5eea7 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 1 Dec 2022 19:25:49 +0800 Subject: [PATCH 046/120] fix:compile error --- source/os/src/osSemaphore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 3a375732b2..2f947d3252 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -75,7 +75,7 @@ int32_t tsem_wait(tsem_t* sem) { return ret; } -int32_t tsem_timewait(tsem_t* sem, int64_t milis) { +int32_t tsem_timewait(tsem_t* sem, int64_t ms) { struct timespec ts; taosClockGetTime(0, &ts); From efcc3b06bb3af6791d670922c290ec5de297d3e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 20:01:09 +0800 Subject: [PATCH 047/120] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + source/libs/executor/src/executil.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7c39e378cb..f86540e3e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3870,6 +3870,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFreeClear(pSupInfo->plist); taosMemoryFree(pSupInfo->colIds); + taosMemoryFree(pSupInfo->slotIds); taosArrayDestroy(pSupInfo->pColAgg); for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 3f62191ca1..6c219f8bd4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1637,7 +1637,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return TSDB_CODE_SUCCESS; } -void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); } +void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); +} int32_t convertFillType(int32_t mode) { int32_t type = TSDB_FILL_NONE; From 1750b5cc92332fd68d551bdfc83fcec7a9ca03f1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Dec 2022 20:34:06 +0800 Subject: [PATCH 048/120] remove invalid table --- source/libs/executor/src/executil.c | 18 +++++++++--------- source/util/src/tarray.c | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8a85885d98..d5bd2c6010 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -438,13 +438,12 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* goto end; } } + removeInvalidTable(uidList, tags); int32_t rows = taosArrayGetSize(uidList); if (rows == 0) { goto end; } - // int64_t stt1 = taosGetTimestampUs(); - // qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); code = blockDataEnsureCapacity(pResBlock, rows); if (code != TSDB_CODE_SUCCESS) { @@ -452,7 +451,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* goto end; } - // int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { int64_t* uid = taosArrayGet(uidList, i); for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) { @@ -467,7 +465,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* #endif } else { void* tag = taosHashGet(tags, uid, sizeof(int64_t)); - ASSERT(tag); + if (tag == NULL) { + continue; + } STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); @@ -923,14 +923,14 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* return -1; } - SArray* pTbList = getTableNameList(pList); - int32_t numOfTables = taosArrayGetSize(pTbList); - SHashObj *uHash = NULL; + SArray* pTbList = getTableNameList(pList); + int32_t numOfTables = taosArrayGetSize(pTbList); + SHashObj* uHash = NULL; size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids if (listlen > 0) { uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < listlen; i++) { - int64_t *uid = taosArrayGet(list, i); + int64_t* uid = taosArrayGet(list, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); } } @@ -2011,4 +2011,4 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag) { char* pBuf = NULL; qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); taosMemoryFree(pBuf); -} \ No newline at end of file +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ae25786375..4bd8294423 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -319,9 +319,9 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) { } else { ASSERT(pSrc->elemSize == sizeof(void*)); - for(int32_t i = 0; i < pSrc->size; ++i) { + for (int32_t i = 0; i < pSrc->size; ++i) { void* p = fn(taosArrayGetP(pSrc, i)); - memcpy(((char*)dst->pData )+ i * dst->elemSize, &p, dst->elemSize); + memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize); } } From 24614d03bd2ef21b454d8b9c558fb29977418960 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 21:35:01 +0800 Subject: [PATCH 049/120] fix(query): set correct sma retrieve procedure. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 42 ++++++++++++++----------- source/libs/executor/src/scanoperator.c | 2 ++ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f86540e3e2..24e9181ae6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -84,7 +84,7 @@ typedef struct SIOCostSummary { typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; - SColumnDataAgg** plist; + SColumnDataAgg** plist; // todo remove this int16_t* colIds; // column ids for loading file block data int16_t* slotIds; // the ordinal index in the destination SSDataBlock int32_t numOfCols; @@ -4041,12 +4041,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - // int64_t stime = taosGetTimestampUs(); - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid); + + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { @@ -4071,7 +4070,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pSup->plist[0] = pTsAgg; // update the number of NULL data rows - size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); + size_t numOfCols = pSup->numOfCols; int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); @@ -4095,43 +4094,48 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } #else + SSDataBlock* pResBlock = pReader->pResBlock; + if (pResBlock->pBlockAgg == NULL) { + size_t num = taosArrayGetSize(pResBlock->pDataBlock); + pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); + } + // fill the all null data column - SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); +// SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); if (pAgg->colId == pSup->colIds[j]) { - taosArrayPush(pNewAggList, pAgg); + pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg; i += 1; j += 1; } else if (pAgg->colId < pSup->colIds[j]) { i += 1; } else if (pSup->colIds[j] < pAgg->colId) { if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { - taosArrayPush(pNewAggList, &pSup->tsColAgg); + pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg; } else { // all date in this block are null SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; - taosArrayPush(pNewAggList, &nullColAgg); + taosArrayPush(pSup->pColAgg, &nullColAgg); + + pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg); } j += 1; } } - taosArrayClear(pSup->pColAgg); - taosArrayAddAll(pSup->pColAgg, pNewAggList); +// taosArrayClear(pSup->pColAgg); - size_t num = taosArrayGetSize(pSup->pColAgg); - for(int32_t k = 0; k < num; ++k) { - pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); - } - - taosArrayDestroy(pNewAggList); +// size_t num = taosArrayGetSize(pSup->pColAgg); +// for(int32_t k = 0; k < num; ++k) { +// pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); +// } #endif pReader->cost.smaDataLoad += 1; - *pBlockStatis = pSup->plist; +// *pBlockStatis = pSup->plist; tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 87aacfdd34..b0104c6025 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -236,6 +236,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, return false; } +#if 0 // if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -256,6 +257,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; } +#endif return true; } From 1faf616f1d089ce592c8dcd56326d7140c75aea0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 22:17:53 +0800 Subject: [PATCH 050/120] fix(query): fix null ptr access in evaluation block distribution. --- source/libs/executor/src/sysscanoperator.c | 37 ++++++++++++---------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 7ac6cee2af..96f21babe6 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1874,27 +1874,30 @@ static void destroyBlockDistScanOperatorInfo(void* param) { } static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { - memset(pCond, 0, sizeof(SQueryTableDataCond)); + memset(pCond, 0, sizeof(SQueryTableDataCond)); - pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = 1; - pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return terrno; - } + pCond->order = TSDB_ORDER_ASC; + pCond->numOfCols = 1; + pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return terrno; + } - pCond->colList->colId = 1; - pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; - pCond->colList->bytes = sizeof(TSKEY); + pCond->colList->colId = 1; + pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; + pCond->colList->bytes = sizeof(TSKEY); - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = uid; - pCond->type = TIMEWINDOW_RANGE_CONTAINED; - pCond->startVersion = -1; - pCond->endVersion = -1; + pCond->pSlotList[0] = 0; - return TSDB_CODE_SUCCESS; + pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->suid = uid; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; + pCond->startVersion = -1; + pCond->endVersion = -1; + + return TSDB_CODE_SUCCESS; } SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, From 03d62e478f4dbac374f2cd15f35f8241d25a6750 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 1 Dec 2022 22:17:57 +0800 Subject: [PATCH 051/120] fix: get prevLogTerm of index zero as 0 only when firstVer is zero --- source/libs/sync/src/syncPipeline.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 03306af925..f0be976402 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -74,7 +74,7 @@ SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, S SyncTerm prevLogTerm = -1; terrno = TSDB_CODE_SUCCESS; - if (prevIndex == -1) return 0; + if (prevIndex == -1 && pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore) == 0) return 0; if (prevIndex > pBuf->matchIndex) { terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; @@ -691,7 +691,6 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod if (pMsg->matchIndex < pNode->pLogBuf->matchIndex) { term = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index + 1); - if (term < 0 || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); if (syncNodeStartSnapshot(pNode, &destId) < 0) { From 3424b184d30b75a8430d352a5238da0fe092ed28 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 1 Dec 2022 04:08:06 +0800 Subject: [PATCH 052/120] refactor(stream): distributed checkpoint --- include/common/tglobal.h | 1 + include/common/tmsg.h | 15 ++ include/common/tmsgdef.h | 10 +- include/libs/stream/tstream.h | 91 +++++--- source/common/src/tglobal.c | 1 + source/common/src/tmsg.c | 25 +++ source/dnode/mnode/impl/inc/mndDef.h | 13 +- source/dnode/mnode/impl/src/mndDef.c | 6 + source/dnode/mnode/impl/src/mndMain.c | 40 +++- source/dnode/mnode/impl/src/mndStream.c | 236 ++++++++++++++------- source/dnode/mnode/impl/src/mndSubscribe.c | 26 +-- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/stream/inc/streamInc.h | 6 +- source/libs/stream/src/streamCheckpoint.c | 194 +++++++++++++++++ source/libs/stream/src/streamMeta.c | 31 +-- source/libs/stream/src/streamRecover.c | 43 ---- 17 files changed, 528 insertions(+), 214 deletions(-) create mode 100644 source/libs/stream/src/streamCheckpoint.c diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ed0f5cd31d..5260713560 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -136,6 +136,7 @@ extern int64_t tsWalFsyncDataSizeLimit; // internal extern int32_t tsTransPullupInterval; extern int32_t tsMqRebalanceInterval; +extern int32_t tsStreamCheckpointTickInterval; extern int32_t tsTtlUnit; extern int32_t tsTtlPushInterval; extern int32_t tsGrantHBInterval; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 80590a25c0..b6db704a7e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1144,6 +1144,13 @@ typedef struct { int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); +typedef struct { + int64_t tick; +} SMStreamTickReq; + +int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); +int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); + typedef struct { int32_t id; uint16_t port; // node sync Port @@ -1744,6 +1751,8 @@ typedef struct { int64_t watermark; int32_t numOfTags; SArray* pTags; // array of SField + // 3.0.20 + int64_t checkpointFreq; // ms } SCMCreateStreamReq; typedef struct { @@ -1943,6 +1952,12 @@ typedef struct { SHashObj* rebSubHash; // SHashObj } SMqDoRebalanceMsg; +typedef struct { + int64_t streamId; + int64_t checkpointId; + char streamName[TSDB_STREAM_FNAME_LEN]; +} SMStreamDoCheckpointMsg; + typedef struct { int64_t status; } SMVSubscribeRsp; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index e80766d249..7833bdf139 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -172,6 +172,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -241,8 +243,11 @@ enum { TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "stream-recover-finish", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "stream-task-check", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT, "stream-checkpoint", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_MON_MSG) @@ -282,6 +287,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 4099551188..0196dcb0a8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -275,31 +275,6 @@ typedef struct { SEpSet epSet; } SStreamChildEpInfo; -typedef struct { - int32_t srcNodeId; - int32_t srcChildId; - int64_t stateSaveVer; - int64_t stateProcessedVer; -} SStreamCheckpointInfo; - -typedef struct { - int64_t streamId; - int64_t checkTs; - int32_t checkpointId; // incremental - int32_t taskId; - SArray* checkpointVer; // SArray -} SStreamMultiVgCheckpointInfo; - -typedef struct { - int32_t taskId; - int32_t checkpointId; // incremental -} SStreamCheckpointKey; - -typedef struct { - int32_t taskId; - SArray* checkpointVer; -} SStreamRecoveringState; - typedef struct SStreamTask { int64_t streamId; int32_t taskId; @@ -364,6 +339,10 @@ typedef struct SStreamTask { int64_t checkReqId; SArray* checkReqIds; // shuffle int32_t refCnt; + + int64_t checkpointingId; + int32_t checkpointAlignCnt; + } SStreamTask; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); @@ -509,6 +488,60 @@ typedef struct { int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq); int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq); +typedef struct { + int64_t streamId; + int64_t checkpointId; + int32_t taskId; + int32_t nodeId; + int64_t expireTime; +} SStreamCheckpointSourceReq; + +typedef struct { + int64_t streamId; + int64_t checkpointId; + int32_t taskId; + int32_t nodeId; + int64_t expireTime; +} SStreamCheckpointSourceRsp; + +int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq); +int32_t tDecodeSStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq); + +int32_t tEncodeSStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp); +int32_t tDecodeSStreamCheckpointSourceRsp(SDecoder* pDecoder, SStreamCheckpointSourceRsp* pRsp); + +typedef struct { + SMsgHead msgHead; + int64_t streamId; + int64_t checkpointId; + int32_t downstreamTaskId; + int32_t downstreamNodeId; + int32_t upstreamTaskId; + int32_t upstreamNodeId; + int32_t childId; + int64_t expireTime; + int8_t taskLevel; +} SStreamCheckpointReq; + +typedef struct { + SMsgHead msgHead; + int64_t streamId; + int64_t checkpointId; + int32_t downstreamTaskId; + int32_t downstreamNodeId; + int32_t upstreamTaskId; + int32_t upstreamNodeId; + int32_t childId; + int64_t expireTime; + int8_t taskLevel; +} SStreamCheckpointRsp; + +int32_t tEncodeSStreamCheckpointReq(SEncoder* pEncoder, const SStreamCheckpointReq* pReq); +int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pReq); + +int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp); +int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp); + typedef struct { int64_t streamId; int32_t downstreamTaskId; @@ -598,18 +631,22 @@ void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen); -int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); -void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId); +void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamLoadTasks(SStreamMeta* pMeta); +// checkpoint +int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); +int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq); +int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp); + #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b6c039f375..0d802f6362 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -166,6 +166,7 @@ int64_t tsWalFsyncDataSizeLimit = (100 * 1024 * 1024L); // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; +int32_t tsStreamCheckpointTickInterval = 1; int32_t tsTtlUnit = 86400; int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c7e98415d1..b264b578fc 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3738,6 +3738,31 @@ int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { return 0; } +int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->tick) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) { if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 04ac5aba49..8e97cb0a23 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -635,6 +635,10 @@ typedef struct { SArray* tasks; // SArray> SSchemaWrapper outputSchema; SSchemaWrapper tagSchema; + + // 3.0.20 + int64_t checkpointFreq; // ms + int64_t currentTick; // do not serialize } SStreamObj; int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); @@ -648,15 +652,6 @@ typedef struct { SArray* childInfo; // SArray } SStreamCheckpointObj; -#if 0 -typedef struct { - int64_t uid; - int64_t streamId; - int8_t status; - int8_t stage; -} SStreamRecoverObj; -#endif - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 2e984212a1..67283c5fce 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -76,6 +76,9 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; + // 3.0.20 + if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1; + tEndEncode(pEncoder); return pEncoder->pos; } @@ -139,6 +142,9 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; + // 3.0.20 + if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1; + tEndDecode(pDecoder); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 2d888d2ff7..1d207dafed 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -85,6 +85,21 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } +static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) { + SMStreamTickReq timerReq = { + .tick = sec, + }; + + int32_t contLen = tSerializeSMStreamTickMsg(NULL, 0, &timerReq); + if (contLen <= 0) return NULL; + void *pReq = rpcMallocCont(contLen); + if (pReq == NULL) return NULL; + + tSerializeSMStreamTickMsg(pReq, contLen, &timerReq); + *pContLen = contLen; + return pReq; +} + static void mndPullupTrans(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); @@ -105,7 +120,24 @@ static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen}; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_TMQ_TIMER, + .pCont = pReq, + .contLen = contLen, + }; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + } +} + +static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { + int32_t contLen = 0; + void *pReq = mndBuildCheckpointTickMsg(&contLen, sec); + if (pReq != NULL) { + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER, + .pCont = pReq, + .contLen = contLen, + }; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } @@ -220,6 +252,12 @@ static void *mndThreadFp(void *param) { mndCalMqRebalance(pMnode); } +#if 0 + if (sec % tsStreamCheckpointTickInterval == 0) { + mndStreamCheckpointTick(pMnode, sec); + } +#endif + if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { mndPullupTelem(pMnode); } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d8cf7a837e..53028caf93 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -36,6 +36,8 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); +static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); +static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); /*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); @@ -62,6 +64,10 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint); + mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); @@ -679,6 +685,164 @@ _OVER: tFreeStreamObj(&streamObj); return code; } +static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SStreamObj *pStream = NULL; + + // iterate all stream obj + while (1) { + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) break; + // incr tick + int64_t currentTick = atomic_add_fetch_64(&pStream->currentTick, 1); + // if >= checkpointFreq, build msg TDMT_MND_STREAM_BEGIN_CHECKPOINT, put into write q + if (currentTick >= pStream->checkpointFreq) { + atomic_store_64(&pStream->currentTick, 0); + SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); + + pMsg->streamId = pStream->uid; + pMsg->checkpointId = tGenIdPI64(); + memcpy(pMsg->streamName, pStream->name, TSDB_STREAM_FNAME_LEN); + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, + .pCont = pMsg, + .contLen = sizeof(SMStreamDoCheckpointMsg), + }; + + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + } + } + + return 0; +} + +static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, const SStreamTask *pTask, + SMStreamDoCheckpointMsg *pMsg) { + SStreamCheckpointSourceReq req = {0}; + req.checkpointId = pMsg->checkpointId; + req.nodeId = pTask->nodeId; + req.expireTime = -1; + req.streamId = pTask->streamId; + req.taskId = pTask->taskId; + + int32_t code; + int32_t blen; + + tEncodeSize(tEncodeSStreamCheckpointSourceReq, &req, blen, code); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t tlen = sizeof(SMsgHead) + blen; + + void *buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + tEncodeSStreamCheckpointSourceReq(&encoder, &req); + + SMsgHead *pMsgHead = (SMsgHead *)buf; + pMsgHead->contLen = htonl(tlen); + pMsgHead->vgId = htonl(pTask->nodeId); + + tEncoderClear(&encoder); + + *pBuf = buf; + *pLen = tlen; + + return 0; +} + +static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + + SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont; + + SStreamObj *pStream = mndAcquireStream(pMnode, pMsg->streamName); + + if (pStream == NULL || pStream->uid != pMsg->streamId) { + mError("start checkpointing failed since stream %s not found", pMsg->streamName); + return -1; + } + + // build new transaction: + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "stream-checkpoint"); + if (pTrans == NULL) return -1; + mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); + taosRLockLatch(&pStream->lock); + // 1. redo action: broadcast checkpoint source msg for all source vg + int32_t totLevel = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < totLevel; i++) { + SArray *pLevel = taosArrayGetP(pStream->tasks, i); + SStreamTask *pTask = taosArrayGetP(pLevel, 0); + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + int32_t sz = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + ASSERT(pTask->nodeId > 0); + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId); + if (pVgObj == NULL) { + ASSERT(0); + taosRUnLockLatch(&pStream->lock); + mndReleaseStream(pMnode, pStream); + mndTransDrop(pTrans); + return -1; + } + + void *buf; + int32_t tlen; + if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask, pMsg) < 0) { + taosRUnLockLatch(&pStream->lock); + mndReleaseStream(pMnode, pStream); + mndTransDrop(pTrans); + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE; + + mndReleaseVgroup(pMnode, pVgObj); + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + taosRUnLockLatch(&pStream->lock); + mndReleaseStream(pMnode, pStream); + mndTransDrop(pTrans); + return -1; + } + } + } + } + // 2. reset tick + atomic_store_64(&pStream->currentTick, 0); + // 3. commit log: stream checkpoint info + taosRUnLockLatch(&pStream->lock); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("failed to prepare trans rebalance since %s", terrstr()); + mndTransDrop(pTrans); + mndReleaseStream(pMnode, pStream); + return -1; + } + + mndReleaseStream(pMnode, pStream); + mndTransDrop(pTrans); + + return 0; +} static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -747,71 +911,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -#if 0 -static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SStreamObj *pStream = NULL; - /*SDbObj *pDb = NULL;*/ - /*SUserObj *pUser = NULL;*/ - - SMRecoverStreamReq recoverReq = {0}; - if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) { - ASSERT(0); - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - pStream = mndAcquireStream(pMnode, recoverReq.name); - - if (pStream == NULL) { - if (recoverReq.igNotExists) { - mInfo("stream:%s, not exist, ignore not exist is set", recoverReq.name); - sdbRelease(pMnode->pSdb, pStream); - return 0; - } else { - terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; - return -1; - } - } - - if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { - return -1; - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); - if (pTrans == NULL) { - mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr()); - sdbRelease(pMnode->pSdb, pStream); - return -1; - } - mInfo("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); - - // broadcast to recover all tasks - if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) { - mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr()); - sdbRelease(pMnode->pSdb, pStream); - return -1; - } - - // update stream status - if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) { - sdbRelease(pMnode->pSdb, pStream); - return -1; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr()); - sdbRelease(pMnode->pSdb, pStream); - mndTransDrop(pTrans); - return -1; - } - - sdbRelease(pMnode->pSdb, pStream); - - return TSDB_CODE_ACTION_IN_PROGRESS; -} -#endif - int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -846,13 +945,6 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { } } -#if 0 - if (mndSetDropOffsetStreamLogs(pMnode, pTrans, pStream) < 0) { - sdbRelease(pSdb, pStream); - goto END; - } -#endif - sdbRelease(pSdb, pStream); } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index ffb46e5f1b..98078f1c60 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -440,9 +440,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "persist-reb"); - mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) return -1; + mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL); // make txn: // 1. redo action: action to all vg @@ -523,28 +523,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); } -#if 0 - if (consumerNum) { - char topic[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true); - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - if (pTopic) { - // TODO make topic complete - SMqTopicObj topicObj = {0}; - memcpy(&topicObj, pTopic, sizeof(SMqTopicObj)); - topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum; - // TODO is that correct? - pTopic->refConsumerCnt = topicObj.refConsumerCnt; - mInfo("subscribe topic %s unref %d consumer cgroup %s, refcnt %d", pTopic->name, consumerNum, cgroup, - topicObj.refConsumerCnt); - if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) { - ASSERT(0); - goto REB_FAIL; - } - } - } -#endif // 4. TODO commit log: modification log diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index aa55204ae5..b133226ed3 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -168,7 +168,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; - streamMetaRemoveTask1(pSnode->pMeta, pReq->taskId); + streamMetaRemoveTask(pSnode->pMeta, pReq->taskId); return 0; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3d9ebec4c9..003b387838 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1425,7 +1425,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - streamMetaRemoveTask1(pTq->pStreamMeta, pReq->taskId); + streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); return 0; } diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 5ff49502df..66496f11f8 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -17,7 +17,6 @@ #define _STREAM_INC_H_ #include "executor.h" -#include "tref.h" #include "tstream.h" #ifdef __cplusplus @@ -25,9 +24,8 @@ extern "C" { #endif typedef struct { - int8_t inited; - int32_t refPool; - void* timer; + int8_t inited; + void* timer; } SStreamGlobalEnv; static SStreamGlobalEnv streamEnv; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c new file mode 100644 index 0000000000..efd19074da --- /dev/null +++ b/source/libs/stream/src/streamCheckpoint.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamInc.h" + +int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->nodeId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->expireTime) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->nodeId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->expireTime) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->nodeId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->expireTime) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamCheckpointSourceRsp(SDecoder* pDecoder, SStreamCheckpointSourceRsp* pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->nodeId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->expireTime) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSStreamCheckpointReq(SEncoder* pEncoder, const SStreamCheckpointReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->expireTime) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->taskLevel) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->expireTime) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->taskLevel) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->upstreamTaskId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->expireTime) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->taskLevel) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->expireTime) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->taskLevel) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +static int32_t streamAlignCheckpoint(SStreamTask* pTask, int64_t checkpointId, int32_t childId) { + if (pTask->checkpointingId == 0) { + pTask->checkpointingId = checkpointId; + pTask->checkpointAlignCnt = taosArrayGetSize(pTask->childEpInfo); + } + + ASSERT(pTask->checkpointingId == checkpointId); + + return atomic_sub_fetch_32(&pTask->checkpointAlignCnt, 1); +} + +static int32_t streamDoCheckpoint(SStreamMeta* pMeta, SStreamTask* pTask, int64_t checkpointId) { + // commit tdb state + streamStateCommit(pTask->pState); + // commit non-tdb state + // copy and save new state + // report to mnode + // send checkpoint req to downstream + return 0; +} + +static int32_t streamDoSourceCheckpoint(SStreamMeta* pMeta, SStreamTask* pTask, int64_t checkpointId) { + // ref wal + // set status checkpointing + // do checkpoint + return 0; +} +int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) { + int32_t code; + int64_t checkpointId = pReq->checkpointId; + + code = streamDoSourceCheckpoint(pMeta, pTask, checkpointId); + if (code < 0) { + // rsp error + return -1; + } + + return 0; +} + +int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq) { + int32_t code; + int64_t checkpointId = pReq->checkpointId; + int32_t childId = pReq->childId; + + if (taosArrayGetSize(pTask->childEpInfo) > 0) { + code = streamAlignCheckpoint(pTask, checkpointId, childId); + if (code > 0) { + return 0; + } + if (code < 0) { + ASSERT(0); + return -1; + } + } + + code = streamDoCheckpoint(pMeta, pTask, checkpointId); + if (code < 0) { + // rsp error + return -1; + } + + // send rsp to all children + + return 0; +} + +int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp) { + // recover step2, scan from wal + // unref wal + // set status normal + return 0; +} diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a864814a74..fc6e1668ba 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -202,7 +202,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { } } -void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId) { +void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { SStreamTask* pTask = *ppTask; @@ -219,35 +219,6 @@ void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId) { } } -int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - SStreamTask* pTask = *ppTask; - taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); - atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); - - if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) { - /*return -1;*/ - } - - if (pTask->triggerParam != 0) { - taosTmrStop(pTask->timer); - } - - while (1) { - int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING); - if (schedStatus != TASK_SCHED_STATUS__ACTIVE) { - tFreeSStreamTask(pTask); - break; - } - taosMsleep(10); - } - } - - return 0; -} - int32_t streamMetaBegin(SStreamMeta* pMeta) { if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 7eee95a580..6889a870d1 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -325,46 +325,3 @@ int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishR tEndDecode(pDecoder); return 0; } - -int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) { - if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1; - if (tEncodeI32(pEncoder, pCheckpoint->srcChildId) < 0) return -1; - if (tEncodeI64(pEncoder, pCheckpoint->stateProcessedVer) < 0) return -1; - return 0; -} - -int32_t tDecodeSStreamCheckpointInfo(SDecoder* pDecoder, SStreamCheckpointInfo* pCheckpoint) { - if (tDecodeI32(pDecoder, &pCheckpoint->srcNodeId) < 0) return -1; - if (tDecodeI32(pDecoder, &pCheckpoint->srcChildId) < 0) return -1; - if (tDecodeI64(pDecoder, &pCheckpoint->stateProcessedVer) < 0) return -1; - return 0; -} - -int32_t tEncodeSStreamMultiVgCheckpointInfo(SEncoder* pEncoder, const SStreamMultiVgCheckpointInfo* pCheckpoint) { - if (tEncodeI64(pEncoder, pCheckpoint->streamId) < 0) return -1; - if (tEncodeI64(pEncoder, pCheckpoint->checkTs) < 0) return -1; - if (tEncodeI32(pEncoder, pCheckpoint->checkpointId) < 0) return -1; - if (tEncodeI32(pEncoder, pCheckpoint->taskId) < 0) return -1; - int32_t sz = taosArrayGetSize(pCheckpoint->checkpointVer); - if (tEncodeI32(pEncoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SStreamCheckpointInfo* pOneVgCkpoint = taosArrayGet(pCheckpoint->checkpointVer, i); - if (tEncodeSStreamCheckpointInfo(pEncoder, pOneVgCkpoint) < 0) return -1; - } - return 0; -} - -int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCheckpointInfo* pCheckpoint) { - if (tDecodeI64(pDecoder, &pCheckpoint->streamId) < 0) return -1; - if (tDecodeI64(pDecoder, &pCheckpoint->checkTs) < 0) return -1; - if (tDecodeI32(pDecoder, &pCheckpoint->checkpointId) < 0) return -1; - if (tDecodeI32(pDecoder, &pCheckpoint->taskId) < 0) return -1; - int32_t sz; - if (tDecodeI32(pDecoder, &sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SStreamCheckpointInfo oneVgCheckpoint; - if (tDecodeSStreamCheckpointInfo(pDecoder, &oneVgCheckpoint) < 0) return -1; - taosArrayPush(pCheckpoint->checkpointVer, &oneVgCheckpoint); - } - return 0; -} From ffb1d52e55500a498501b2c5aee2a881f9d76952 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 07:57:22 +0800 Subject: [PATCH 053/120] fix: compatility after update stream --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndDef.c | 7 ++++--- source/dnode/mnode/impl/src/mndStream.c | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 8e97cb0a23..ad40c10822 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -642,7 +642,7 @@ typedef struct { } SStreamObj; int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); -int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj); +int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver); void tFreeStreamObj(SStreamObj* pObj); typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 67283c5fce..b5c2fb05b3 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -83,7 +83,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { return pEncoder->pos; } -int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { +int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; @@ -143,8 +143,9 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; // 3.0.20 - if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1; - + if (sver >= 2) { + if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1; + } tEndDecode(pDecoder); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 53028caf93..5fa27fd062 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_STREAM_VER_NUMBER 1 +#define MND_STREAM_VER_NUMBER 2 #define MND_STREAM_RESERVE_SIZE 64 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); @@ -131,7 +131,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER; - if (sver != MND_STREAM_VER_NUMBER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto STREAM_DECODE_OVER; } @@ -152,7 +152,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { SDecoder decoder; tDecoderInit(&decoder, buf, tlen + 1); - if (tDecodeSStreamObj(&decoder, pStream) < 0) { + if (tDecodeSStreamObj(&decoder, pStream, 2) < 0) { tDecoderClear(&decoder); goto STREAM_DECODE_OVER; } From 99657ccf788d42976a9c0a1d4549f314a6f56681 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 08:00:10 +0800 Subject: [PATCH 054/120] fix: compatibility after update stream --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 6ac8db80c7..7ee688d220 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -153,7 +153,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { SDecoder decoder; tDecoderInit(&decoder, buf, tlen + 1); - if (tDecodeSStreamObj(&decoder, pStream, 2) < 0) { + if (tDecodeSStreamObj(&decoder, pStream, sver) < 0) { tDecoderClear(&decoder); goto STREAM_DECODE_OVER; } From 67d6ea93b06006cbb0b9af9f0a44e4c3448a0da6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 2 Dec 2022 10:02:22 +0800 Subject: [PATCH 055/120] enh: ins_table count optimize --- source/libs/parser/src/parTranslater.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index becf58b401..307eefe2bd 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -47,6 +47,7 @@ typedef struct STranslateContext { SParseMetaCache* pMetaCache; bool createStream; bool stableQuery; + bool showRewrite; } STranslateContext; typedef struct SFullDatabaseName { @@ -2222,8 +2223,8 @@ static int32_t getVnodeSysTableVgroupListImpl(STranslateContext* pCxt, SName* pT if (TSDB_DB_NAME_T == pTargetName->type) { int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); - if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || - TSDB_CODE_MND_DB_IN_DROPPING == code) { + if (!pCxt->showRewrite && (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code)) { // system table query should not report errors code = TSDB_CODE_SUCCESS; } @@ -6294,6 +6295,7 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { code = createShowCondition((SShowStmt*)pQuery->pRoot, pStmt); } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; @@ -6347,6 +6349,7 @@ static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) { code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags); } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; pSelect->tagScan = true; nodesDestroyNode(pQuery->pRoot); @@ -6377,6 +6380,7 @@ static int32_t rewriteShowDnodeVariables(STranslateContext* pCxt, SQuery* pQuery } } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pSelect; @@ -6396,6 +6400,7 @@ static int32_t rewriteShowVnodes(STranslateContext* pCxt, SQuery* pQuery) { } } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; @@ -6437,6 +6442,7 @@ static int32_t rewriteShowTableDist(STranslateContext* pCxt, SQuery* pQuery) { code = nodesListMakeStrictAppend(&pStmt->pProjectionList, createBlockDistFunc()); } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; From db42ec46bfbf2ac4e416ef7735393eed844d3fb4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 10:05:52 +0800 Subject: [PATCH 056/120] refact: adust error code while dnode startup --- include/libs/qcom/query.h | 7 +++-- include/util/taoserror.h | 29 +++++++++++++++---- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 1 + source/dnode/mgmt/node_mgmt/src/dmTransport.c | 13 +++++++-- source/dnode/mnode/impl/src/mndMain.c | 12 -------- source/util/src/terror.c | 5 ++++ 6 files changed, 44 insertions(+), 23 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f51aa88485..a97368ffee 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -264,11 +264,12 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later -#define NEED_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ +#define NEED_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK) + (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 25d37020cc..6f9fa7623d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -40,21 +40,37 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc +// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x +// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x +// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x +// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x +// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //2.x +// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //2.x +// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //2.x #define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) +// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //2.x +// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //2.x +// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //2.x +// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //2.x +// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //2.x +// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x +// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x +#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) +// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) #define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) #define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) //common & util -#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) - #define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) #define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102) +// #define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x #define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) @@ -69,7 +85,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) #define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) -#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) +#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) // #define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) #define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) @@ -81,7 +97,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) #define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120) -#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) +#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) // #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0122) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0123) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0124) @@ -94,6 +110,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B) #define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C) +#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) // +#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) // + //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) #define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 07a612bb35..30ef7b9542 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -144,6 +144,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dGError("msg:%p, not processed in mgmt queue", pMsg); break; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 95656fd76c..1bd7846d1c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -51,13 +51,15 @@ static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { } int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { + const STraceId *trace = &pMsg->info.traceId; + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; if (msgFp == NULL) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dGError("msg:%p, not processed since no handler", pMsg); return -1; } - const STraceId *trace = &pMsg->info.traceId; dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); pMsg->info.wrapper = pWrapper; return (*msgFp)(pWrapper->pMgmt, pMsg); @@ -99,18 +101,23 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dmProcessServerStartupStatus(pDnode, pRpc); return; } else { - terrno = TSDB_CODE_APP_NOT_READY; + if (pDnode->status == DND_STAT_INIT) { + terrno = TSDB_CODE_APP_IS_STARTING; + } else { + terrno = TSDB_CODE_APP_IS_STOPPING; + } goto _OVER; } } - if (IsReq(pRpc) && pRpc->pCont == NULL) { + if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) { dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; goto _OVER; } if (pHandle->defaultNtype == NODE_END) { + dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType)); terrno = TSDB_CODE_MSG_NOT_PROCESSED; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 9f4c2e048f..d46ae7aaa9 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -644,17 +644,6 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { return -1; } -static int32_t mndCheckMsgContent(SRpcMsg *pMsg) { - if (!IsReq(pMsg)) return 0; - if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0; - - const STraceId *trace = &pMsg->info.traceId; - mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen, - pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); - terrno = TSDB_CODE_INVALID_MSG_LEN; - return -1; -} - int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; const STraceId *trace = &pMsg->info.traceId; @@ -666,7 +655,6 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { return -1; } - if (mndCheckMsgContent(pMsg) != 0) return -1; if (mndCheckMnodeState(pMsg) != 0) return -1; mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2025f196f2..3e8c016a74 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -67,6 +67,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, "Invalid Ref ID") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, "Ref is already there") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") + TAOS_DEFINE_ERROR(TSDB_CODE_APP_ERROR, "Unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE, "Out of range") @@ -83,6 +84,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VERSION_NUMBER, "Invalid version numbe TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VERSION_STRING, "Invalid version string") TAOS_DEFINE_ERROR(TSDB_CODE_VERSION_NOT_COMPATIBLE, "Version not compatible") TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") + TAOS_DEFINE_ERROR(TSDB_CODE_COMPRESS_ERROR, "Failed to compress msg") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") @@ -97,6 +99,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISKSPACE, "Out of disk space") TAOS_DEFINE_ERROR(TSDB_CODE_TIMEOUT_ERROR, "Operation timeout") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting up") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down") + //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, "Invalid qhandle") From 83782fb27bf679b44353511e0eb70393e4193902 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 10:07:48 +0800 Subject: [PATCH 057/120] fix(query): fix invalid tag fill and clear nullbitmp --- include/common/tdatablock.h | 6 ++--- source/common/src/tdatablock.c | 8 +++---- source/libs/executor/src/tfill.c | 28 +++++++++++++++++----- tests/script/tsim/parser/lastrow_query.sim | 4 ++-- tests/script/tsim/valgrind/checkUdf.sim | 3 ++- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 4208d85648..e1d3b01611 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,9 +41,9 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -#define colDataClearNull_f(bm_, r_) \ - do { \ - BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ +#define colDataClearNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \ } while (0) #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c300c3e6f0..e9be89a68b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1679,15 +1679,15 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } else { // reset the bitmap value - int32_t stopIndex = BitmapLen(n) * 8; - for(int32_t i = n + 1; i < stopIndex; ++i) { + /*int32_t stopIndex = BitmapLen(n) * 8; + for(int32_t i = n; i < stopIndex; ++i) { colDataClearNull_f(pColInfoData->nullbitmap, i); } int32_t remain = BitmapLen(total) - BitmapLen(n); if (remain > 0) { - memset(pColInfoData->nullbitmap, 0, remain); - } + memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain); + }*/ } } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 42b1d058b5..4c8967744e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -60,8 +60,18 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd if (pCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex); if (!filled) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; - SGroupKeys* pKey = taosArrayGet(p, i); + SRowVal* p = NULL; + if (FILL_IS_ASC_FILL(pFillInfo)) { + if (pFillInfo->prev.key != 0) { + p = &pFillInfo->prev; // prev has been set value + } else { // otherwise, use the value in the next row + p = &pFillInfo->next; + } + } else { + p = &pFillInfo->next; + } + + SGroupKeys* pKey = taosArrayGet(p->pRowVal, i); doSetVal(pDstColInfo, rowIndex, pKey); } } else { @@ -261,7 +271,10 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) { +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) { + SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); + pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex]; + for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType; if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) { @@ -272,7 +285,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); - saveColData(pRow, i, p, isNull); + saveColData(pRowVal->pRowVal, i, p, isNull); } else { ASSERT(0); } @@ -296,7 +309,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the next value for interpolation if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next.pRowVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && @@ -318,7 +331,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { int32_t nextRowIndex = pFillInfo->index + 1; - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, pFillInfo->next.pRowVal); + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next); } // copy rows to dst buffer @@ -334,6 +347,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (!colDataIsNull_s(pSrc, pFillInfo->index)) { colDataAppend(pDst, index, src, false); saveColData(pFillInfo->prev.pRowVal, i, src, false); + if (pFillInfo->srcTsSlotId == dstSlotId) { + pFillInfo->prev.key = *(int64_t*)src; + } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false); diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index 5f557fd7bd..9ca9d407b8 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -70,10 +70,10 @@ sql select _wstart, t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00 if $row != 2 then return -1 endi -if $data01 != NULL then +if $data01 != 8 then return -1 endi -if $data02 != NULL then +if $data02 != 8 then return -1 endi if $data03 != NULL then diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim index 594d649714..caf316bd86 100644 --- a/tests/script/tsim/valgrind/checkUdf.sim +++ b/tests/script/tsim/valgrind/checkUdf.sim @@ -121,12 +121,13 @@ if $data01 != 152.420471066 then return -1 endi -sql select udf2(f2) from udf.t2 group by 1-udf1(f1); +sql select udf2(f2) from udf.t2 group by 1-udf1(f1) order by 1-udf1(f1) print $rows , $data00 , $data10 if $rows != 2 then return -1 endi if $data00 != 2.000000000 then + print expect 2.000000000 , actual: $data00 return -1 endi if $data10 != 12.083045974 then From 64a2bd481cb259825e3bd993102ee679f1e9fd50 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 2 Dec 2022 10:30:23 +0800 Subject: [PATCH 058/120] fix: check subquery time series error --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d2d97f7b90..15682a2cfe 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -750,8 +750,8 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { return (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId); } else if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { SFunctionNode* pFunc = (SFunctionNode*)pExpr; - if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_FIRST == pFunc->funcType || - FUNCTION_TYPE_LAST == pFunc->funcType) { + if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType || + FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) { return true; From 1fcd99176bc71614842f0de54f47828de8178dc8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 10:59:52 +0800 Subject: [PATCH 059/120] fix: filter memory leak cause of same const strings --- source/libs/scalar/inc/filterInt.h | 11 +++-- source/libs/scalar/src/filter.c | 65 ++++++++++++++++-------------- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index e26bc59d47..2023d38777 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -101,6 +101,11 @@ typedef int32_t (*filter_desc_compare_func)(const void *, const void *); typedef bool (*filter_exec_func)(void *, int32_t, SColumnInfoData *, SColumnDataAgg *, int16_t, int32_t *); typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char *, void **); +typedef struct SFilterDataInfo { + int32_t idx; + void* addr; +} SFilterDataInfo; + typedef struct SFilterRangeCompare { int64_t s; int64_t e; @@ -294,9 +299,9 @@ struct SFilterInfo { #define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) #define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) -#define FILTER_GET_FLAG(st, f) (st & f) -#define FILTER_SET_FLAG(st, f) st |= (f) -#define FILTER_CLR_FLAG(st, f) st &= (~f) +#define FILTER_GET_FLAG(st, f) ((st) & (f)) +#define FILTER_SET_FLAG(st, f) (st) |= (f) +#define FILTER_CLR_FLAG(st, f) (st) &= (~f) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) #define FLT_PACKAGE_UNIT_HASH_KEY(v, op1, op2, lidx, ridx, ridx2) \ diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 45931c209c..ef92377410 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -963,16 +963,17 @@ int32_t filterGetFiledByDesc(SFilterFields *fields, int32_t type, void *v) { return -1; } -int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { +int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen, bool *sameBuf) { if (type == FLD_TYPE_VALUE) { if (info->pctx.valHash == false) { qError("value hash is empty"); return -1; } - void *hv = taosHashGet(info->pctx.valHash, v, dataLen); - if (hv) { - return *(int32_t *)hv; + SFilterDataInfo *dInfo = taosHashGet(info->pctx.valHash, v, dataLen); + if (dInfo) { + *sameBuf = (dInfo->addr == v); + return dInfo->idx; } } @@ -982,9 +983,10 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d // In the params, we should use void *data instead of void **data, there is no need to use taosMemoryFreeClear(*data) to // set *data = 0 Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, - bool freeIfExists) { + bool freeIfExists, int16_t *srcFlag) { int32_t idx = -1; uint32_t *num; + bool sameBuf = false; num = &info->fields[type].num; @@ -992,7 +994,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, if (type == FLD_TYPE_COLUMN) { idx = filterGetFiledByDesc(&info->fields[type], type, desc); } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { - idx = filterGetFiledByData(info, type, *data, dataLen); + idx = filterGetFiledByData(info, type, *data, dataLen, &sameBuf); } } @@ -1020,11 +1022,14 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); } - taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); + SFilterDataInfo dInfo = {idx, *data}; + taosHashPut(info->pctx.valHash, *data, dataLen, &dInfo, sizeof(dInfo)); } - } else { - if (data && freeIfExists) { + } else if (data) { + if (freeIfExists) { taosMemoryFreeClear(*data); + } else if (sameBuf) { + FILTER_SET_FLAG(*srcFlag, FLD_DATA_NO_FREE); } } @@ -1035,7 +1040,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, } static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { - filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); + filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false, NULL); FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); @@ -1064,7 +1069,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f v = node; } - filterAddField(info, v, NULL, type, fid, 0, true); + filterAddField(info, v, NULL, type, fid, 0, true, NULL); return TSDB_CODE_SUCCESS; } @@ -1195,7 +1200,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { len = tDataTypes[type].bytes; - filterAddField(info, NULL, (void **)&out.columnData->pData, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, (void **)&out.columnData->pData, FLD_TYPE_VALUE, &right, len, true, NULL); out.columnData->pData = NULL; } else { void *data = taosMemoryCalloc(1, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); // reserved space for simple_copy @@ -1203,7 +1208,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } memcpy(data, nodesGetValueFromNode(valueNode), tDataTypes[type].bytes); - filterAddField(info, NULL, (void **)&data, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, (void **)&data, FLD_TYPE_VALUE, &right, len, true, NULL); } filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); @@ -1234,28 +1239,26 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit *u uint8_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = 0; - filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); + filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false, NULL); SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); if (u->right.type == FLD_TYPE_VALUE) { void *data = FILTER_UNIT_VAL_DATA(src, u); + SFilterField *rField = FILTER_UNIT_RIGHT_FIELD(src, u); + if (IS_VAR_DATA_TYPE(type)) { if (FILTER_UNIT_OPTR(u) == OP_TYPE_IN) { filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, - false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. + false, &rField->flag); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false, &rField->flag); } } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false, &rField->flag); } - - flag = FLD_DATA_NO_FREE; - t = FILTER_UNIT_RIGHT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); } else { pright = NULL; } @@ -1313,17 +1316,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (func(&ra->s, &ra->e) == 0) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &ra->e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true, NULL); filterAddUnitImpl( dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, @@ -1337,7 +1340,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1346,7 +1349,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1393,16 +1396,16 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (func(&r->ra.s, &r->ra.e) == 0) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &r->ra.e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true, NULL); filterAddUnitImpl( dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, @@ -1421,7 +1424,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1430,7 +1433,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); From dafc1ac826da01b26f51b6d1b4e5549b7365c3df Mon Sep 17 00:00:00 2001 From: sunpeng Date: Fri, 2 Dec 2022 12:03:01 +0800 Subject: [PATCH 060/120] doc: update grafana version in English doc and modifythe way grafana plugin installed in application doc (#18635) --- docs/en/20-third-party/01-grafana.mdx | 2 +- docs/en/25-application/01-telegraf.md | 10 ++-------- docs/en/25-application/02-collectd.md | 10 ++-------- docs/zh/25-application/01-telegraf.md | 10 ++-------- docs/zh/25-application/02-collectd.md | 10 ++-------- 5 files changed, 9 insertions(+), 33 deletions(-) diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx index e0fbefd5a8..ca32ce8afc 100644 --- a/docs/en/20-third-party/01-grafana.mdx +++ b/docs/en/20-third-party/01-grafana.mdx @@ -76,7 +76,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource You can also download zip files from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) and install manually. The commands are as follows: ```bash -GF_VERSION=3.2.2 +GF_VERSION=3.2.7 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip # from Grafana diff --git a/docs/en/25-application/01-telegraf.md b/docs/en/25-application/01-telegraf.md index f700326449..65fb08ee67 100644 --- a/docs/en/25-application/01-telegraf.md +++ b/docs/en/25-application/01-telegraf.md @@ -38,15 +38,9 @@ Download the latest TDengine-server from the [Downloads](http://tdengine.com/en/ ## Data Connection Setup -### Download TDengine plug-in to grafana plug-in directory +### Install Grafana Plugin and Configure Data Source -```bash -1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip -2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/ -3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine -4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini -5. sudo systemctl restart grafana-server.service -``` +Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source) ### Modify /etc/telegraf/telegraf.conf diff --git a/docs/en/25-application/02-collectd.md b/docs/en/25-application/02-collectd.md index 692cd8d929..97412b2309 100644 --- a/docs/en/25-application/02-collectd.md +++ b/docs/en/25-application/02-collectd.md @@ -41,15 +41,9 @@ Download the latest TDengine-server from the [Downloads](http://tdengine.com/en/ ## Data Connection Setup -### Copy the TDengine plugin to the grafana plugin directory +### Install Grafana Plugin and Configure Data Source -```bash -1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip -2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/ -3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine -4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini -5. sudo systemctl restart grafana-server.service -``` +Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source) ### Configure collectd diff --git a/docs/zh/25-application/01-telegraf.md b/docs/zh/25-application/01-telegraf.md index 6338264d17..ec263d7792 100644 --- a/docs/zh/25-application/01-telegraf.md +++ b/docs/zh/25-application/01-telegraf.md @@ -39,15 +39,9 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据，例如 ## 数据链路设置 -### 下载 TDengine 插件到 Grafana 插件目录 +### 安装 Grafana Plugin 并配置数据源 -```bash -1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip -2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/ -3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine -4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini -5. sudo systemctl restart grafana-server.service -``` +请参考[安装 Grafana Plugin 并配置数据源](/third-party/grafana/#%E5%AE%89%E8%A3%85-grafana-plugin-%E5%B9%B6%E9%85%8D%E7%BD%AE%E6%95%B0%E6%8D%AE%E6%BA%90)。 ### 修改 /etc/telegraf/telegraf.conf diff --git a/docs/zh/25-application/02-collectd.md b/docs/zh/25-application/02-collectd.md index c6230f48ab..8b39a6431d 100644 --- a/docs/zh/25-application/02-collectd.md +++ b/docs/zh/25-application/02-collectd.md @@ -41,15 +41,9 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据，例如 ## 数据链路设置 -### 复制 TDengine 插件到 grafana 插件目录 +### 安装 Grafana Plugin 并配置数据源 -```bash -1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip -2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/ -3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine -4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini -5. sudo systemctl restart grafana-server.service -``` +请参考[安装 Grafana Plugin 并配置数据源](/third-party/grafana/#%E5%AE%89%E8%A3%85-grafana-plugin-%E5%B9%B6%E9%85%8D%E7%BD%AE%E6%95%B0%E6%8D%AE%E6%BA%90)。 ### 配置 collectd From 37299a53f8cb6eb42c504a2884c06959f4916b88 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 2 Dec 2022 12:14:56 +0800 Subject: [PATCH 061/120] feat: sql command 'show user privileges' --- source/dnode/mnode/impl/inc/mndUser.h | 2 +- source/dnode/mnode/impl/src/mndShow.c | 2 ++ source/dnode/mnode/impl/src/mndUser.c | 34 +++++++++++++++++++++------ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 970d1db7db..8cd1950daf 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -30,7 +30,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); -SHashObj *mndDupDbHash(SHashObj *pOld); +SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 20c2ebb0a4..ec2e844ba5 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -108,6 +108,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_APPS; } else if (strncasecmp(name, TSDB_INS_TABLE_STREAM_TASKS, len) == 0) { type = TSDB_MGMT_TABLE_STREAM_TASKS; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) { + type = TSDB_MGMT_TABLE_PRIVILEGES; } else { // ASSERT(0); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5f34adce77..1666fe2f09 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -161,7 +161,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { SDB_SET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER); - db = taosHashIterate(pUser->topics, topic); + topic = taosHashIterate(pUser->topics, topic); } SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) @@ -446,7 +446,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc return 0; } -SHashObj *mndDupDbHash(SHashObj *pOld) { +SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen) { SHashObj *pNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pNew == NULL) { @@ -457,7 +457,7 @@ SHashObj *mndDupDbHash(SHashObj *pOld) { char *db = taosHashIterate(pOld, NULL); while (db != NULL) { int32_t len = strlen(db) + 1; - if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) { + if (taosHashPut(pNew, db, len, db, dataLen) != 0) { taosHashCancelIterate(pOld, db); taosHashCleanup(pNew); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -517,9 +517,9 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { newUser.updateTime = taosGetTimestampMs(); taosRLockLatch(&pUser->lock); - newUser.readDbs = mndDupDbHash(pUser->readDbs); - newUser.writeDbs = mndDupDbHash(pUser->writeDbs); - newUser.topics = mndDupDbHash(pUser->topics); + newUser.readDbs = mndDupObjHash(pUser->readDbs, TSDB_DB_FNAME_LEN); + newUser.writeDbs = mndDupObjHash(pUser->writeDbs, TSDB_DB_FNAME_LEN); + newUser.topics = mndDupObjHash(pUser->topics, TSDB_TOPIC_FNAME_LEN); taosRUnLockLatch(&pUser->lock); if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) { @@ -832,6 +832,26 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfTopics = taosHashGetSize(pUser->topics); if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break; + if (pUser->superUser) { + cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); + colDataAppend(pColInfo, numOfRows, (const char *)userName, false); + + char privilege[20] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(privilege, "all", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)privilege, false); + + char objName[20] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)objName, false); + + numOfRows++; + } + char *db = taosHashIterate(pUser->readDbs, NULL); while (db != NULL) { cols = 0; @@ -902,7 +922,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); numOfRows++; - db = taosHashIterate(pUser->topics, topic); + topic = taosHashIterate(pUser->topics, topic); } sdbRelease(pSdb, pUser); From 4f8c04cf79aca8bbe14a9ebb953717e27e9dd9c8 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Fri, 2 Dec 2022 12:47:19 +0800 Subject: [PATCH 062/120] doc: writting from kafka (#18626) * doc: writting from kafka * rename file * docs: typo fix Co-authored-by: Shuduo Sang --- .../03-insert-data/20-kafka-writting.mdx | 46 +++++ ...influxdb-line.mdx => 30-influxdb-line.mdx} | 0 ...tsdb-telnet.mdx => 40-opentsdb-telnet.mdx} | 0 ...opentsdb-json.mdx => 50-opentsdb-json.mdx} | 0 .../{05-high-volume.md => 60-high-volume.md} | 0 .../07-develop/03-insert-data/_py_kafka.mdx | 60 ++++++ docs/examples/python/kafka_example.py | 192 ++++++++++++++++++ .../03-insert-data/20-kafka-writting.mdx | 47 +++++ ...influxdb-line.mdx => 30-influxdb-line.mdx} | 0 ...tsdb-telnet.mdx => 40-opentsdb-telnet.mdx} | 0 ...opentsdb-json.mdx => 50-opentsdb-json.mdx} | 0 .../{05-high-volume.md => 60-high-volume.md} | 0 .../07-develop/03-insert-data/_py_kafka.mdx | 60 ++++++ 13 files changed, 405 insertions(+) create mode 100644 docs/en/07-develop/03-insert-data/20-kafka-writting.mdx rename docs/en/07-develop/03-insert-data/{02-influxdb-line.mdx => 30-influxdb-line.mdx} (100%) rename docs/en/07-develop/03-insert-data/{03-opentsdb-telnet.mdx => 40-opentsdb-telnet.mdx} (100%) rename docs/en/07-develop/03-insert-data/{04-opentsdb-json.mdx => 50-opentsdb-json.mdx} (100%) rename docs/en/07-develop/03-insert-data/{05-high-volume.md => 60-high-volume.md} (100%) create mode 100644 docs/en/07-develop/03-insert-data/_py_kafka.mdx create mode 100644 docs/examples/python/kafka_example.py create mode 100644 docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx rename docs/zh/07-develop/03-insert-data/{02-influxdb-line.mdx => 30-influxdb-line.mdx} (100%) rename docs/zh/07-develop/03-insert-data/{03-opentsdb-telnet.mdx => 40-opentsdb-telnet.mdx} (100%) rename docs/zh/07-develop/03-insert-data/{04-opentsdb-json.mdx => 50-opentsdb-json.mdx} (100%) rename docs/zh/07-develop/03-insert-data/{05-high-volume.md => 60-high-volume.md} (100%) create mode 100644 docs/zh/07-develop/03-insert-data/_py_kafka.mdx diff --git a/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx new file mode 100644 index 0000000000..ffb969a8a6 --- /dev/null +++ b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx @@ -0,0 +1,46 @@ +--- +title: Write from Kafka +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; +import PyKafka from "./_py_kafka.mdx"; + +## About Kafka + +Apache Kafka is an open-source distributed event streaming platform, used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications. For the key concepts of kafka, please refer to [kafka documentation](https://kafka.apache.org/documentation/#gettingStarted). + +### kafka topic + +Messages in Kafka are organized by topics. A topic may have one or more partitions. We can manage kafka topics through `kafka-topics`. + +create a topic named `kafka-events`: + +``` +bin/kafka-topics.sh --create --topic kafka-events --bootstrap-server localhost:9092 +``` + +Alter `kafka-events` topic to set partitions to 3: + +``` +bin/kafka-topics.sh --alter --topic kafka-events --partitions 3 --bootstrap-server=localhost:9092 +``` + +Show all topics and partitions in Kafka: + +``` +bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe +``` + +## Insert into TDengine + +We can write data into TDengine via SQL or Schemaless. For more information, please refer to [Insert Using SQL](/develop/insert-data/sql-writing/) or [High Performance Writing](/develop/insert-data/high-volume/) or [Schemaless Writing](/reference/schemaless/). + +## Examples + + + + + + + diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx similarity index 100% rename from docs/en/07-develop/03-insert-data/02-influxdb-line.mdx rename to docs/en/07-develop/03-insert-data/30-influxdb-line.mdx diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx similarity index 100% rename from docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx rename to docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx similarity index 100% rename from docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx rename to docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx diff --git a/docs/en/07-develop/03-insert-data/05-high-volume.md b/docs/en/07-develop/03-insert-data/60-high-volume.md similarity index 100% rename from docs/en/07-develop/03-insert-data/05-high-volume.md rename to docs/en/07-develop/03-insert-data/60-high-volume.md diff --git a/docs/en/07-develop/03-insert-data/_py_kafka.mdx b/docs/en/07-develop/03-insert-data/_py_kafka.mdx new file mode 100644 index 0000000000..dc43a0d415 --- /dev/null +++ b/docs/en/07-develop/03-insert-data/_py_kafka.mdx @@ -0,0 +1,60 @@ +### python Kafka 客户端 + +For python kafka client, please refer to [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python). In this document, we use [kafka-python](http://github.com/dpkp/kafka-python). + +### consume from Kafka + +The simple way to consume messages from Kafka is to read messages one by one. The demo is as follows: + +``` +from kafka import KafkaConsumer +consumer = KafkaConsumer('my_favorite_topic') +for msg in consumer: + print (msg) +``` + +For higher performance, we can consume message from kafka in batch. The demo is as follows: + +``` +from kafka import KafkaConsumer +consumer = KafkaConsumer('my_favorite_topic') +while True: + msgs = consumer.poll(timeout_ms=500, max_records=1000) + if msgs: + print (msgs) +``` + +### multi-threading + +For more higher performance we can process data from kafka in multi-thread. We can use python's ThreadPoolExecutor to achieve multithreading. The demo is as follows: + +``` +from concurrent.futures import ThreadPoolExecutor, Future +pool = ThreadPoolExecutor(max_workers=10) +pool.submit(...) +``` + +### multi-process + +For more higher performance, sometimes we use multiprocessing. In this case, the number of Kafka Consumers should not be greater than the number of Kafka Topic Partitions. The demo is as follows: + +``` +from multiprocessing import Process + +ps = [] +for i in range(5): + p = Process(target=Consumer().consume()) + p.start() + ps.append(p) + +for p in ps: + p.join() +``` + +In addition to python's built-in multithreading and multiprocessing library, we can also use the third-party library gunicorn. + +### Examples + +```py +{{#include docs/examples/python/kafka_example.py}} +``` diff --git a/docs/examples/python/kafka_example.py b/docs/examples/python/kafka_example.py new file mode 100644 index 0000000000..735059eec0 --- /dev/null +++ b/docs/examples/python/kafka_example.py @@ -0,0 +1,192 @@ +#! encoding = utf-8 +import json +import time +from json import JSONDecodeError +from typing import Callable +import logging +from concurrent.futures import ThreadPoolExecutor, Future + +import taos +from kafka import KafkaConsumer +from kafka.consumer.fetcher import ConsumerRecord + + +class Consumer(object): + DEFAULT_CONFIGS = { + 'kafka_brokers': 'localhost:9092', + 'kafka_topic': 'python_kafka', + 'kafka_group_id': 'taos', + 'taos_host': 'localhost', + 'taos_user': 'root', + 'taos_password': 'taosdata', + 'taos_database': 'power', + 'taos_port': 6030, + 'timezone': None, + 'clean_after_testing': False, + 'bath_consume': True, + 'batch_size': 1000, + 'async_model': True, + 'workers': 10 + } + + LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose', + 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', + 'California.SantaClara', 'California.Cupertino'] + + CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1' + USE_DATABASE_SQL = 'use {}' + DROP_TABLE_SQL = 'drop table if exists meters' + DROP_DATABASE_SQL = 'drop database if exists {}' + CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) ' \ + 'tags (location binary(64), groupId int)' + CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})' + INSERT_SQL_HEADER = "insert into " + INSERT_PART_SQL = 'power.{} values (\'{}\', {}, {}, {})' + + def __init__(self, **configs): + self.config: dict = self.DEFAULT_CONFIGS + self.config.update(configs) + self.consumer = KafkaConsumer( + self.config.get('kafka_topic'), # topic + bootstrap_servers=self.config.get('kafka_brokers'), + group_id=self.config.get('kafka_group_id'), + ) + self.taos = taos.connect( + host=self.config.get('taos_host'), + user=self.config.get('taos_user'), + password=self.config.get('taos_password'), + port=self.config.get('taos_port'), + timezone=self.config.get('timezone'), + ) + if self.config.get('async_model'): + self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers')) + self.tasks: list[Future] = [] + # tags and table mapping # key: {location}_{groupId} value: + self.tag_table_mapping = {} + i = 0 + for location in self.LOCATIONS: + for j in range(1, 11): + table_name = 'd{}'.format(i) + self._cache_table(location=location, group_id=j, table_name=table_name) + i += 1 + + def init_env(self): + # create database and table + self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database'))) + self.taos.execute(self.CREATE_DATABASE_SQL.format(self.config.get('taos_database'))) + self.taos.execute(self.USE_DATABASE_SQL.format(self.config.get('taos_database'))) + self.taos.execute(self.DROP_TABLE_SQL) + self.taos.execute(self.CREATE_STABLE_SQL) + for tags, table_name in self.tag_table_mapping.items(): + location, group_id = _get_location_and_group(tags) + self.taos.execute(self.CREATE_TABLE_SQL.format(table_name, location, group_id)) + + def consume(self): + logging.warning('## start consumer topic-[%s]', self.config.get('kafka_topic')) + try: + if self.config.get('bath_consume'): + self._run_batch(self._to_taos_batch) + else: + self._run(self._to_taos) + except KeyboardInterrupt: + logging.warning("## caught keyboard interrupt, stopping") + finally: + self.stop() + + def stop(self): + # close consumer + if self.consumer is not None: + self.consumer.commit() + self.consumer.close() + + # multi thread + if self.config.get('async_model'): + for task in self.tasks: + while not task.done(): + pass + if self.pool is not None: + self.pool.shutdown() + + # clean data + if self.config.get('clean_after_testing'): + self.taos.execute(self.DROP_TABLE_SQL) + self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database'))) + # close taos + if self.taos is not None: + self.taos.close() + + def _run(self, f: Callable[[ConsumerRecord], bool]): + for message in self.consumer: + if self.config.get('async_model'): + self.pool.submit(f(message)) + else: + f(message) + + def _run_batch(self, f: Callable[[list[list[ConsumerRecord]]], None]): + while True: + messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size')) + if messages: + if self.config.get('async_model'): + self.pool.submit(f, messages.values()) + else: + f(list(messages.values())) + if not messages: + time.sleep(0.1) + + def _to_taos(self, message: ConsumerRecord) -> bool: + sql = self.INSERT_SQL_HEADER + self._build_sql(message.value) + if len(sql) == 0: # decode error, skip + return True + logging.info('## insert sql %s', sql) + return self.taos.execute(sql=sql) == 1 + + def _to_taos_batch(self, messages: list[list[ConsumerRecord]]): + sql = self._build_sql_batch(messages=messages) + if len(sql) == 0: # decode error, skip + return + self.taos.execute(sql=sql) + + def _build_sql(self, msg_value: str) -> str: + try: + data = json.loads(msg_value) + except JSONDecodeError as e: + logging.error('## decode message [%s] error ', msg_value, e) + return '' + location = data.get('location') + group_id = data.get('groupId') + ts = data.get('ts') + current = data.get('current') + voltage = data.get('voltage') + phase = data.get('phase') + + table_name = self._get_table_name(location=location, group_id=group_id) + return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase) + + def _build_sql_batch(self, messages: list[list[ConsumerRecord]]) -> str: + sql_list = [] + for partition_messages in messages: + for message in partition_messages: + sql_list.append(self._build_sql(message.value)) + + return self.INSERT_SQL_HEADER + ' '.join(sql_list) + + def _cache_table(self, location: str, group_id: int, table_name: str): + self.tag_table_mapping[_tag_table_mapping_key(location=location, group_id=group_id)] = table_name + + def _get_table_name(self, location: str, group_id: int) -> str: + return self.tag_table_mapping.get(_tag_table_mapping_key(location=location, group_id=group_id)) + + +def _tag_table_mapping_key(location: str, group_id: int): + return '{}_{}'.format(location, group_id) + + +def _get_location_and_group(key: str) -> (str, int): + fields = key.split('_') + return fields[0], fields[1] + + +if __name__ == '__main__': + consumer = Consumer(async_model=True) + consumer.init_env() + consumer.consume() \ No newline at end of file diff --git a/docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx b/docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx new file mode 100644 index 0000000000..32d3c2e5cb --- /dev/null +++ b/docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx @@ -0,0 +1,47 @@ +--- +title: 从 Kafka 写入 +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; +import PyKafka from "./_py_kafka.mdx"; + +## Kafka 介绍 + +Apache Kafka 是开源的分布式消息分发平台，被广泛应用于高性能数据管道、流式数据分析、数据集成和事件驱动类型的应用程序。Kafka 包含 Producer、Consumer 和 Topic，其中 Producer 是向 Kafka 发送消息的进程，Consumer 是从 Kafka 消费消息的进程。Kafka 相关概念可以参考[官方文档](https://kafka.apache.org/documentation/#gettingStarted)。 + + +### kafka topic + +Kafka 的消息按 topic 组织，每个 topic 会有一到多个 partition。可以通过 kafka 的 `kafka-topics` 管理 topic。 + +创建名为 `kafka-events` 的topic: + +``` +bin/kafka-topics.sh --create --topic kafka-events --bootstrap-server localhost:9092 +``` + +修改 `kafka-events` 的 partition 数量为 3: + +``` +bin/kafka-topics.sh --alter --topic kafka-events --partitions 3 --bootstrap-server=localhost:9092 +``` + +展示所有的 topic 和 partition: + +``` +bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe +``` + +## 写入 TDengine + +TDengine 支持 Sql 方式和 Schemaless 方式的数据写入，Sql 方式数据写入可以参考 [TDengine SQL 写入](/develop/insert-data/sql-writing/) 和 [TDengine 高效写入](/develop/insert-data/high-volume/)。Schemaless 方式数据写入可以参考 [TDengine Schemaless 写入](/reference/schemaless/) 文档。 + +## 示例代码 + + + + + + + diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx similarity index 100% rename from docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx rename to docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx similarity index 100% rename from docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx rename to docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx similarity index 100% rename from docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx rename to docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx diff --git a/docs/zh/07-develop/03-insert-data/05-high-volume.md b/docs/zh/07-develop/03-insert-data/60-high-volume.md similarity index 100% rename from docs/zh/07-develop/03-insert-data/05-high-volume.md rename to docs/zh/07-develop/03-insert-data/60-high-volume.md diff --git a/docs/zh/07-develop/03-insert-data/_py_kafka.mdx b/docs/zh/07-develop/03-insert-data/_py_kafka.mdx new file mode 100644 index 0000000000..cd7edf557d --- /dev/null +++ b/docs/zh/07-develop/03-insert-data/_py_kafka.mdx @@ -0,0 +1,60 @@ +### python Kafka 客户端 + +Kafka 的 python 客户端可以参考文档 [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python)。推荐使用 [confluent-kafka-python](https://github.com/confluentinc/confluent-kafka-python) 和 [kafka-python](http://github.com/dpkp/kafka-python)。以下示例以 [kafka-python](http://github.com/dpkp/kafka-python) 为例。 + +### 从 Kafka 消费数据 + +Kafka 客户端采用 pull 的方式从 Kafka 消费数据，可以采用单条消费的方式或批量消费的方式读取数据。使用 [kafka-python](http://github.com/dpkp/kafka-python) 客户端单条消费数据的示例如下： + +``` +from kafka import KafkaConsumer +consumer = KafkaConsumer('my_favorite_topic') +for msg in consumer: + print (msg) +``` + +单条消费的方式在数据流量大的情况下往往存在性能瓶颈，导致 Kafka 消息积压，更推荐使用批量消费的方式消费数据。使用 [kafka-python](http://github.com/dpkp/kafka-python) 客户端批量消费数据的示例如下： + +``` +from kafka import KafkaConsumer +consumer = KafkaConsumer('my_favorite_topic') +while True: + msgs = consumer.poll(timeout_ms=500, max_records=1000) + if msgs: + print (msgs) +``` + +### Python 多线程 + +为了提高数据写入效率，通常采用多线程的方式写入数据，可以使用 python 线程池 ThreadPoolExecutor 实现多线程。示例代码如下： + +``` +from concurrent.futures import ThreadPoolExecutor, Future +pool = ThreadPoolExecutor(max_workers=10) +pool.submit(...) +``` + +### Python 多进程 + +单个python进程不能充分发挥多核 CPU 的性能，有时候我们会选择多进程的方式。在多进程的情况下，需要注意，Kafka Consumer 的数量应该小于等于 Kafka Topic Partition 数量。Python 多进程示例代码如下： + +``` +from multiprocessing import Process + +ps = [] +for i in range(5): + p = Process(target=Consumer().consume()) + p.start() + ps.append(p) + +for p in ps: + p.join() +``` + +除了 Python 内置的多线程和多进程方式，还可以通过第三方库 gunicorn 实现并发。 + +### 完整示例 + +```py +{{#include docs/examples/python/kafka_example.py}} +``` From 83d33ceae27b93d78008159a9e2b865dfa81f5c7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 2 Dec 2022 13:12:41 +0800 Subject: [PATCH 063/120] feat: sql command 'show user privileges' --- source/dnode/mnode/impl/inc/mndUser.h | 2 +- source/dnode/mnode/impl/src/mndUser.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 8cd1950daf..ff43d69d2f 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -30,7 +30,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); -SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen); +SHashObj *mndDupDbHash(SHashObj *pOld, int32_t dataLen); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1666fe2f09..291b83c80e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -446,7 +446,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc return 0; } -SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen) { +SHashObj *mndDupDbHash(SHashObj *pOld, int32_t dataLen) { SHashObj *pNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pNew == NULL) { @@ -517,9 +517,9 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { newUser.updateTime = taosGetTimestampMs(); taosRLockLatch(&pUser->lock); - newUser.readDbs = mndDupObjHash(pUser->readDbs, TSDB_DB_FNAME_LEN); - newUser.writeDbs = mndDupObjHash(pUser->writeDbs, TSDB_DB_FNAME_LEN); - newUser.topics = mndDupObjHash(pUser->topics, TSDB_TOPIC_FNAME_LEN); + newUser.readDbs = mndDupDbHash(pUser->readDbs, TSDB_DB_FNAME_LEN); + newUser.writeDbs = mndDupDbHash(pUser->writeDbs, TSDB_DB_FNAME_LEN); + newUser.topics = mndDupDbHash(pUser->topics, TSDB_TOPIC_FNAME_LEN); taosRUnLockLatch(&pUser->lock); if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) { From 71140f565e928855bb7c18d53cd810273c283096 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 2 Dec 2022 13:43:48 +0800 Subject: [PATCH 064/120] feat: sql command 'show user privileges' --- source/dnode/mnode/impl/inc/mndUser.h | 3 ++- source/dnode/mnode/impl/src/mndUser.c | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index ff43d69d2f..cf7deba397 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -30,7 +30,8 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); -SHashObj *mndDupDbHash(SHashObj *pOld, int32_t dataLen); +SHashObj *mndDupDbHash(SHashObj *pOld); +SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 291b83c80e..806ba0c98e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -446,7 +446,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc return 0; } -SHashObj *mndDupDbHash(SHashObj *pOld, int32_t dataLen) { +SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen) { SHashObj *pNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pNew == NULL) { @@ -469,6 +469,10 @@ SHashObj *mndDupDbHash(SHashObj *pOld, int32_t dataLen) { return pNew; } +SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAME_LEN); } + +SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); } + static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -517,9 +521,9 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { newUser.updateTime = taosGetTimestampMs(); taosRLockLatch(&pUser->lock); - newUser.readDbs = mndDupDbHash(pUser->readDbs, TSDB_DB_FNAME_LEN); - newUser.writeDbs = mndDupDbHash(pUser->writeDbs, TSDB_DB_FNAME_LEN); - newUser.topics = mndDupDbHash(pUser->topics, TSDB_TOPIC_FNAME_LEN); + newUser.readDbs = mndDupDbHash(pUser->readDbs); + newUser.writeDbs = mndDupDbHash(pUser->writeDbs); + newUser.topics = mndDupTopicHash(pUser->topics); taosRUnLockLatch(&pUser->lock); if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) { From 7bb30150a9ef13f3f5c66d9b06fbf9e9f73d88a4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 14:23:36 +0800 Subject: [PATCH 065/120] fix: fix memory leak --- source/libs/qworker/src/qwMsg.c | 2 +- source/libs/scalar/src/filter.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index d9a7cea411..7dc13ce5f3 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -336,7 +336,7 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo * } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - taosMemoryFree(msg); + rpcFreeCont(msg); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ef92377410..1cf4bd9923 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1024,8 +1024,11 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterDataInfo dInfo = {idx, *data}; taosHashPut(info->pctx.valHash, *data, dataLen, &dInfo, sizeof(dInfo)); + if (srcFlag) { + FILTER_SET_FLAG(*srcFlag, FLD_DATA_NO_FREE); + } } - } else if (data) { + } else if (type != FLD_TYPE_COLUMN && data) { if (freeIfExists) { taosMemoryFreeClear(*data); } else if (sameBuf) { From 42a512c39fa8a456496d1b154dc5be22b1bbb1bf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 15:52:32 +0800 Subject: [PATCH 066/120] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 22 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 176 +++++++++++------------- source/libs/executor/src/executor.c | 18 ++- source/libs/executor/src/scanoperator.c | 37 ++--- tests/system-test/2-query/unique.py | 6 +- 5 files changed, 116 insertions(+), 143 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 53b8eadb33..64c15c6c0e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -161,17 +161,17 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); -void tsdbReaderClose(STsdbReader *pReader); -bool tsdbNextDataBlock(STsdbReader *pReader); -void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); -SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); -int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -void *tsdbGetIdx(SMeta *pMeta); -void *tsdbGetIvtIdx(SMeta *pMeta); -uint64_t getReaderMaxVersion(STsdbReader *pReader); +void tsdbReaderClose(STsdbReader *pReader); +bool tsdbNextDataBlock(STsdbReader *pReader); +void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave); +SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +void *tsdbGetIdx(SMeta *pMeta); +void *tsdbGetIvtIdx(SMeta *pMeta); +uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void **pReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 24e9181ae6..859ae026a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -81,12 +81,15 @@ typedef struct SIOCostSummary { double createScanInfoList; } SIOCostSummary; +typedef struct SColInfo { + int16_t colId; + int16_t slotId; +} SColInfo; + typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; - SColumnDataAgg** plist; // todo remove this - int16_t* colIds; // column ids for loading file block data - int16_t* slotIds; // the ordinal index in the destination SSDataBlock + SColInfo* colInfo; // column ids for loading file block data int32_t numOfCols; char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. bool smaValid; // the sma on all queried columns are activated @@ -159,6 +162,7 @@ struct STsdbReader { STsdb* pTsdb; uint64_t suid; int16_t order; + bool freeBlock; STimeWindow window; // the primary query time window that applies to all queries SSDataBlock* pResBlock; int32_t capacity; @@ -218,22 +222,21 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); - pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES); - pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); - if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) { - taosMemoryFree(pSupInfo->colIds); - taosMemoryFree(pSupInfo->buildBuf); - taosMemoryFree(pSupInfo->slotIds); + pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES)); + if (pSupInfo->colInfo == NULL) { + taosMemoryFree(pSupInfo->colInfo); return TSDB_CODE_OUT_OF_MEMORY; } + pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { - pSupInfo->colIds[i] = pCols[i].colId; - pSupInfo->slotIds[i] = pSlotIdList[i]; + pSupInfo->colInfo[i].colId = pCols[i].colId; + pSupInfo->colInfo[i].slotId = pSlotIdList[i]; if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + } else { + pSupInfo->buildBuf[i] = NULL; } } @@ -245,7 +248,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; - if (pTCol->colId == pSupInfo->colIds[j]) { + if (pTCol->colId == pSupInfo->colInfo[j].colId) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; return; @@ -253,7 +256,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) i += 1; j += 1; - } else if (pTCol->colId < pSupInfo->colIds[j]) { + } else if (pTCol->colId < pSupInfo->colInfo[j].colId) { // do nothing i += 1; } else { @@ -455,7 +458,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb if (pLReader->pInfo == NULL) { // here we ignore the first column, which is always be the primary timestamp column pLReader->pInfo = - tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1); + tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); if (pLReader->pInfo == NULL) { tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); return terrno; @@ -594,14 +597,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket ASSERT(pCond->numOfCols > 0); + if (pReader->pResBlock == NULL) { + pReader->freeBlock = true; + pReader->pResBlock = createResBlock(pCond, pReader->capacity); + if (pReader->pResBlock == NULL) { + code = terrno; + goto _end; + } + } + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg)); - pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES); - if (pSup->pColAgg == NULL || pSup->plist == NULL) { + if (pSup->pColAgg == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -1083,8 +1094,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); - if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1095,10 +1106,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn rowIndex = 0; SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pSupInfo->colIds[i]) { + if (pData->cid < pSupInfo->colInfo[i].colId) { colIndex += 1; - } else if (pData->cid == pSupInfo->colIds[i]) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + } else if (pData->cid == pSupInfo->colInfo[i].colId) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); @@ -1116,7 +1127,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1124,7 +1135,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1162,7 +1173,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); TABLEID tid = {.suid = pReader->suid, .uid = uid}; int32_t code = - tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1); + tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1621,7 +1632,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2493,7 +2504,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]); + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -3542,24 +3553,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { - col_id_t colId = pSupInfo->colIds[i]; + col_id_t colId = pSupInfo->colInfo[i].colId; if (colId == pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; @@ -3570,7 +3581,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3586,8 +3597,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } @@ -3598,13 +3609,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols && j < numOfInputCols) { SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - if (pData->cid < pSupInfo->colIds[i]) { + if (pData->cid < pSupInfo->colInfo[i].colId) { j += 1; continue; } - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]); - if (pData->cid == pSupInfo->colIds[i]) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pData->cid == pSupInfo->colInfo[i].colId) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3617,7 +3628,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNULL(pCol, outputRowIndex); i += 1; } @@ -3723,7 +3734,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pCond->twindows.ekey -= 1; } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr); + int32_t capacity = 0; + if (pResBlock == NULL) { + capacity = 4096; + } else { + capacity = pResBlock->info.capacity; + } + + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3868,10 +3886,6 @@ void tsdbReaderClose(STsdbReader* pReader) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - taosMemoryFreeClear(pSupInfo->plist); - taosMemoryFree(pSupInfo->colIds); - taosMemoryFree(pSupInfo->slotIds); - taosArrayDestroy(pSupInfo->pColAgg); for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { if (pSupInfo->buildBuf[i] != NULL) { @@ -3879,9 +3893,12 @@ void tsdbReaderClose(STsdbReader* pReader) { } } - taosMemoryFree(pSupInfo->buildBuf); - tBlockDataDestroy(&pReader->status.fileBlockData, true); + if (pReader->freeBlock) { + pReader->pResBlock = blockDataDestroy(pReader->pResBlock); + } + taosMemoryFree(pSupInfo->colInfo); + tBlockDataDestroy(&pReader->status.fileBlockData, true); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -4025,18 +4042,18 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64 } } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockSMA, bool* allHave) { int32_t code = 0; *allHave = false; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } // there is no statistics data for composed block if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } @@ -4054,7 +4071,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS return code; } } else { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } @@ -4067,32 +4084,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; pTsAgg->min = pReader->pResBlock->info.window.skey; pTsAgg->max = pReader->pResBlock->info.window.ekey; - pSup->plist[0] = pTsAgg; // update the number of NULL data rows size_t numOfCols = pSup->numOfCols; int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); -#if 0 - while (j < numOfCols && i < size) { - SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colIds[j]) { - if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { - pSup->plist[j] = pAgg; - } else { - *allHave = false; - break; - } - i += 1; - j += 1; - } else if (pAgg->colId < pSup->colIds[j]) { - i += 1; - } else if (pSup->colIds[j] < pAgg->colId) { - j += 1; - } - } -#else SSDataBlock* pResBlock = pReader->pResBlock; if (pResBlock->pBlockAgg == NULL) { @@ -4100,52 +4097,40 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); } - // fill the all null data column -// SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); - while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colIds[j]) { - pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg; + if (pAgg->colId == pSup->colInfo[j].colId) { + pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg; i += 1; j += 1; - } else if (pAgg->colId < pSup->colIds[j]) { + } else if (pAgg->colId < pSup->colInfo[j].colId) { i += 1; - } else if (pSup->colIds[j] < pAgg->colId) { - if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { - pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg; + } else if (pSup->colInfo[j].colId < pAgg->colId) { + if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg; } else { // all date in this block are null - SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; + SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow}; taosArrayPush(pSup->pColAgg, &nullColAgg); - pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg); + pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg); } j += 1; } } -// taosArrayClear(pSup->pColAgg); - -// size_t num = taosArrayGetSize(pSup->pColAgg); -// for(int32_t k = 0; k < num; ++k) { -// pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); -// } - -#endif - + *pBlockSMA = pResBlock->pBlockAgg; pReader->cost.smaDataLoad += 1; -// *pBlockStatis = pSup->plist; tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } -static SArray* doRetrieveDataBlock(STsdbReader* pReader) { +static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (pStatus->composedDataBlock) { - return pReader->pResBlock->pDataBlock; + return pReader->pResBlock; } SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); @@ -4166,10 +4151,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { } copyBlockDataToSDataBlock(pReader, pBlockScanInfo); - return pReader->pResBlock->pDataBlock; + return pReader->pResBlock; } -SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { +SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { return doRetrieveDataBlock(pReader->innerReader[0]); @@ -4196,7 +4181,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { // allocate buffer in order to load data blocks from file memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); - memset(pReader->suppInfo.plist, 0, POINTER_BYTES); pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; tsdbDataFReaderClose(&pReader->pFileReader); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index d675ac05d5..df11add919 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -957,21 +957,27 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s pCond->order = TSDB_ORDER_ASC; pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return terrno; } - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->twindows = TSWINDOW_INITIALIZER; pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; - pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; - pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; + SColumnInfo* pColInfo = &pCond->colList[i]; + pColInfo->type = pMtInfo->schema->pSchema[i].type; + pColInfo->bytes = pMtInfo->schema->pSchema[i].bytes; + pColInfo->colId = pMtInfo->schema->pSchema[i].colId; + + pCond->pSlotList[i] = i; } return TSDB_CODE_SUCCESS; @@ -1116,7 +1122,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); ASSERT(size == 1); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b0104c6025..f09b7dae58 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -224,10 +224,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA } static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + bool allColumnsHaveAgg = true; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -387,11 +385,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (pCols == NULL) { + SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + if (p == NULL) { return terrno; } + ASSERT(p == pBlock); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value @@ -994,19 +993,10 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU return NULL; } - bool hasBlock = tsdbNextDataBlock(pReader); - if (hasBlock) { - SDataBlockInfo* pBInfo = &pBlock->info; - -// int32_t rows = 0; -// tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); - - SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - -// relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); + if (tsdbNextDataBlock(pReader)) { + /*SSDataBlock* p = */tsdbRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); - - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid); + pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); } tsdbReaderClose(pReader); @@ -2030,20 +2020,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { - SSDataBlock* pBlock = &pInfo->pRes; - if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window); - pBlock->info.rows = rows; - - SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); - pBlock->pDataBlock = pCols; - if (pCols == NULL) { + SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); + if (pBlock == NULL) { longjmp(pTaskInfo->env, terrno); } diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 2b0336d2d7..6af9b130ef 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -429,10 +429,10 @@ class TDTestCase: tdSql.checkRows(2) # nest query - tdSql.query(f"select unique(c1) from (select _rowts , t1 ,c1 , tbname from {dbname}.stb1 ) ") + tdSql.query(f"select unique(c1) v from (select _rowts , t1 ,c1 , tbname from {dbname}.stb1 ) order by v") tdSql.checkRows(11) - tdSql.checkData(0,0,6) - tdSql.checkData(10,0,3) + tdSql.checkData(1,0,0) + tdSql.checkData(10,0,9) tdSql.query(f"select unique(t1) from (select _rowts , t1 , tbname from {dbname}.stb1 )") tdSql.checkRows(2) tdSql.checkData(0,0,4) From 13f36ec20a4185fb238284e1f93953bbc553dde8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 16:06:10 +0800 Subject: [PATCH 067/120] refact: remove TSDB_CODE_APP_NOT_READY and TSDB_CODE_NODE_NOT_DEPLOYED --- include/libs/qcom/query.h | 13 ++-- include/util/taoserror.h | 45 ++++++++++- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 6 +- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 29 ++++++- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 16 +++- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 14 ++-- source/dnode/mgmt/test/mnode/dmnode.cpp | 10 +-- source/dnode/mgmt/test/qnode/dqnode.cpp | 8 +- source/dnode/mgmt/test/snode/dsnode.cpp | 8 +- source/dnode/mgmt/test/vnode/vnode.cpp | 4 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndInfoSchema.c | 4 +- source/dnode/mnode/impl/src/mndMain.c | 69 +++++++++++----- source/dnode/mnode/impl/src/mndMnode.c | 10 +-- source/dnode/mnode/impl/src/mndPerfSchema.c | 4 +- source/dnode/mnode/impl/src/mndQnode.c | 6 +- source/dnode/mnode/impl/src/mndSma.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 6 +- source/dnode/mnode/impl/src/mndSync.c | 29 +++---- source/dnode/mnode/impl/src/mndTrans.c | 9 ++- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 4 +- source/dnode/vnode/src/vnd/vnodeSync.c | 26 ++++--- source/libs/function/src/udfd.c | 4 +- source/libs/sync/src/syncMain.c | 78 ++++++++++--------- source/util/src/terror.c | 19 +++-- 26 files changed, 279 insertions(+), 152 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index a97368ffee..9d54ce6aee 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -266,19 +266,20 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ - (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ + (_code) == TSDB_CODE_APP_IS_STARTING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ (_type) == TDMT_MND_DROP_STB) -#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \ +#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_MNODE_NOT_FOUND || \ SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ - SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_APP_NOT_READY) + SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_SYN_RESTORING || \ + (_code) == TSDB_CODE_APP_IS_STARTING) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6f9fa7623d..b00d2d25fb 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -59,7 +59,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x // #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x #define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) +// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //2.x #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) // #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) @@ -298,6 +298,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) #define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D4) #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) +#define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) // mnode-mq @@ -330,13 +331,49 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482) // dnode -#define TSDB_CODE_NODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408) -#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0409) -#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040A) +// #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x +// #define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) // 2.x +// #define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0402) // 2.x +// #define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) // 2.x +// #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) // 2.x +// #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) // 2.x +// #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) // 2.x +// #define TSDB_CODE_DND_VNODE_OPEN_FAILED TAOS_DEF_ERROR_CODE(0, 0x0407) // 2.x +#define TSDB_CODE_DNODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408) +#define TSDB_CODE_MNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0409) +#define TSDB_CODE_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040A) +#define TSDB_CODE_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040B) +#define TSDB_CODE_QNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040C) +#define TSDB_CODE_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040D) +#define TSDB_CODE_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040E) +#define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040F) +#define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0410) +#define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) // vnode +// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x +// #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x +// #define TSDB_CODE_VND_ACTION_NEED_REPROCESS. TAOS_DEF_ERROR_CODE(0, 0x0502) // 2.x #define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) +// #define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // 2.x +// #define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) // 2.x +// #define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) // 2.x +// #define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) // 2.x +// #define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) // 2.x +// #define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) // 2.x +// #define TSDB_CODE_VND_INVALID_VRESION_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) // 2.x +// #define TSDB_CODE_VND_IS_FULL TAOS_DEF_ERROR_CODE(0, 0x050B) // 2.x +// #define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) // 2.x +// #define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) // 2.x +// #define TSDB_CODE_VND_IS_BALANCING TAOS_DEF_ERROR_CODE(0, 0x050E) // 2.x +// #define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) // 2.x +// #define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) // 2.x #define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) +// #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) // 2.x +// #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) // 2.x +// #define TSDB_CODE_WAIT_THREAD_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0515) // 2.x +#define TSDB_CODE_VND_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0520) // internal +#define TSDB_CODE_VND_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0521) // internal #define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0522) #define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0524) #define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0525) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e15d7ac3df..bc46772858 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -216,7 +216,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dDebug("vgId:%d, already exist", req.vgId); tFreeSCreateVnodeReq(&req); vmReleaseVnode(pMgmt, pVnode); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + terrno = TSDB_CODE_VND_ALREADY_EXIST; code = terrno; return 0; } @@ -307,7 +307,7 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dError("vgId:%d, failed to alter replica since %s", vgId, terrstr()); - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + terrno = TSDB_CODE_VND_NOT_EXIST; return -1; } @@ -369,7 +369,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + terrno = TSDB_CODE_VND_NOT_EXIST; return -1; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 421d723202..e3bda5a3f0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -152,7 +152,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { dmReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + switch (ntype) { + case MNODE: + terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED; + break; + case QNODE: + terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED; + break; + case SNODE: + terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED; + break; + default: + terrno = TSDB_CODE_APP_ERROR; + } dError("failed to create node since %s", terrstr()); return -1; } @@ -191,7 +203,20 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + switch (ntype) { + case MNODE: + terrno = TSDB_CODE_MNODE_NOT_DEPLOYED; + break; + case QNODE: + terrno = TSDB_CODE_QNODE_NOT_DEPLOYED; + break; + case SNODE: + terrno = TSDB_CODE_SNODE_NOT_DEPLOYED; + break; + default: + terrno = TSDB_CODE_APP_ERROR; + } + dError("failed to drop node since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 2c9020b668..02a268afda 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -189,7 +189,6 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); // dTrace("node:%s, is acquired, ref:%d", pWrapper->name, refCount); } else { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; pRetWrapper = NULL; } taosThreadRwlockUnlock(&pWrapper->lock); @@ -205,7 +204,20 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); // dTrace("node:%s, is marked, ref:%d", pWrapper->name, refCount); } else { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + switch (pWrapper->ntype) { + case MNODE: + terrno = TSDB_CODE_MNODE_NOT_FOUND; + break; + case QNODE: + terrno = TSDB_CODE_QNODE_NOT_FOUND; + break; + case SNODE: + terrno = TSDB_CODE_SNODE_NOT_FOUND; + break; + default: + terrno = TSDB_CODE_APP_IS_STOPPING; + break; + } code = -1; } taosThreadRwlockUnlock(&pWrapper->lock); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1bd7846d1c..1bfa1cd70f 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -172,8 +172,7 @@ _OVER: if (IsReq(pRpc)) { SRpcMsg rsp = {.code = code, .info = pRpc->info}; - if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && - pRpc->msgType < TDMT_VND_MSG) { + if (code == TSDB_CODE_MNODE_NOT_FOUND) { dmBuildMnodeRedirectRsp(pDnode, &rsp); } @@ -226,7 +225,11 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; - terrno = TSDB_CODE_NODE_OFFLINE; + if (pDnode->status == DND_STAT_INIT) { + terrno = TSDB_CODE_APP_IS_STARTING; + } else { + terrno = TSDB_CODE_APP_IS_STOPPING; + } dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); return -1; } else { @@ -240,8 +243,9 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); } static bool rpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_MNODE_NOT_FOUND) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 7b7eb97216..28cb376b37 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED); } { @@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED); } { @@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED); } } @@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_NOT_DEPLOYED); } { @@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index a2c6a2c28c..3beb57c516 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED); } } @@ -120,7 +120,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED); } test.Restart(); @@ -135,7 +135,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index e3ad65d831..30d6a34813 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED); } } @@ -120,7 +120,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED); } test.Restart(); @@ -135,7 +135,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index 8a8e332289..24e7affd88 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -63,7 +63,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { ASSERT_EQ(pRsp->code, 0); test.Restart(); } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_VND_ALREADY_EXIST); } } } @@ -285,7 +285,7 @@ TEST_F(DndTestVnode, 06_Drop_Vnode) { ASSERT_EQ(pRsp->code, 0); test.Restart(); } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_VND_NOT_EXIST); } } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 250a302cc4..df25e44bee 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -787,7 +787,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !dropReq.force) { if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 09172115f8..82294ac7bf 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -71,7 +71,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) { int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo, STableMetaRsp *pRsp) { if (NULL == pMnode->infosMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } @@ -97,7 +97,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { if (NULL == pMnode->infosMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d46ae7aaa9..eb6742a564 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -45,7 +45,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) { int32_t code = 0; taosThreadRwlockRdlock(&pMnode->lock); if (pMnode->stopped) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_IS_STOPPING; code = -1; } else if (!mndIsLeader(pMnode)) { code = -1; @@ -599,11 +599,45 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } - if (mndAcquireRpc(pMsg->info.node) == 0) return 0; - SMnode *pMnode = pMsg->info.node; + SMnode *pMnode = pMsg->info.node; + taosThreadRwlockRdlock(&pMnode->lock); + if (pMnode->stopped) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_APP_IS_STOPPING; + return -1; + } + + terrno = 0; SSyncState state = syncGetState(pMnode->syncMgmt.sync); + if (terrno != 0) { + taosThreadRwlockUnlock(&pMnode->lock); + return -1; + } + if (state.state != TAOS_SYNC_STATE_LEADER) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_NOT_LEADER; + goto _OVER; + } + + if (!state.restored || !pMnode->restored) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_RESTORING; + goto _OVER; + } + +#if 1 + atomic_add_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is acquired, ref:%d", ref); +#endif + + taosThreadRwlockUnlock(&pMnode->lock); + return 0; + +_OVER: if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { @@ -616,29 +650,24 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { SEpSet epSet = {0}; mndGetMnodeEpSet(pMnode, &epSet); - mDebug( + mGDebug( "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " "role:%s, redirect numOfEps:%d inUse:%d", pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored, syncStr(state.restored), epSet.numOfEps, epSet.inUse); - if (epSet.numOfEps > 0) { - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - } + if (epSet.numOfEps <= 0) return -1; - int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); - pMsg->info.rsp = rpcMallocCont(contLen); - if (pMsg->info.rsp != NULL) { - tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); - pMsg->info.hasEpSet = 1; - pMsg->info.rspLen = contLen; - terrno = TSDB_CODE_RPC_REDIRECT; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } - } else { - terrno = TSDB_CODE_APP_NOT_READY; + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + } + + int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); + pMsg->info.rsp = rpcMallocCont(contLen); + if (pMsg->info.rsp != NULL) { + tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); + pMsg->info.hasEpSet = 1; + pMsg->info.rspLen = contLen; } return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 2620100b6c..9c847c1138 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -292,7 +292,7 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p .pCont = pReq, .contLen = contLen, .msgType = TDMT_DND_CREATE_MNODE, - .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, + .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED, }; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -333,7 +333,7 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop .pCont = pReq, .contLen = contLen, .msgType = TDMT_DND_DROP_MNODE, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, + .acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED, }; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -440,7 +440,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; goto _OVER; } @@ -490,7 +490,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (totalMnodes == 2) { if (force) { mError("cant't force drop dnode, since a mnode on it and replica is 2"); - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; return -1; } mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); @@ -574,7 +574,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index e874c1a53e..cf463304a1 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -68,7 +68,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) { int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { if (NULL == pMnode->perfsMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } @@ -94,7 +94,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { if (NULL == pMnode->perfsMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 26e7f72cf7..28a5dee2db 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -205,7 +205,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_QNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -232,7 +232,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -345,7 +345,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 7be2d48c70..f9d0a9b9ba 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -463,7 +463,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -780,7 +780,7 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_NOT_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a31801f376..e6a253bcc0 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -210,7 +210,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_SNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -237,7 +237,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -352,7 +352,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 10cd6416b4..3e56081043 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -318,7 +318,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { if (pMgmt->transId != 0) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosWUnLockLatch(&pMgmt->lock); - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; return -1; } @@ -339,13 +339,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID); code = 0; } else { - mInfo("trans:%d, failed to proposed since %s", transId, terrstr()); + mError("trans:%d, failed to proposed since %s", transId, terrstr()); taosWLockLatch(&pMgmt->lock); pMgmt->transId = 0; taosWUnLockLatch(&pMgmt->lock); - if (terrno == TSDB_CODE_SYN_NOT_LEADER) { - terrno = TSDB_CODE_APP_NOT_READY; - } else { + if (terrno == 0) { terrno = TSDB_CODE_APP_ERROR; } } @@ -381,20 +379,23 @@ void mndSyncStop(SMnode *pMnode) { } bool mndIsLeader(SMnode *pMnode) { + terrno = 0; SSyncState state = syncGetState(pMnode->syncMgmt.sync); - if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) { - if (state.state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } - mDebug("vgId:1, mnode not ready, state:%s, restore:%d", syncStr(state.state), state.restored); + if (terrno != 0) { + mDebug("vgId:1, mnode is stopping"); return false; } - if (!mndGetRestored(pMnode)) { - terrno = TSDB_CODE_APP_NOT_READY; + if (state.state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + mDebug("vgId:1, mnode not leader, state:%s", syncStr(state.state)); + return false; + } + + if (!state.restored || !pMnode->restored) { + terrno = TSDB_CODE_SYN_RESTORING; + mDebug("vgId:1, mnode not restored:%d:%d", state.restored, pMnode->restored); return false; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 27c58dfba1..38bfe7f6cb 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -918,10 +918,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && ((code == TSDB_CODE_APP_NOT_READY && pTrans->failedTimes > 60) || - (code != TSDB_CODE_APP_NOT_READY && pTrans->failedTimes > 6))) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_APP_IS_STARTING) { + if (pTrans->failedTimes > 60) sendRsp = true; + } else { + if (pTrans->failedTimes > 6) sendRsp = true; + } if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; - sendRsp = true; } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 544215a905..161bf7536f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -998,7 +998,7 @@ int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -1098,7 +1098,7 @@ int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_NOT_EXIST; if (isRedo) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 60be7cfbc0..89c2b6931a 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -173,7 +173,7 @@ class MndTestTrans2 : public ::testing::Test { action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED; mndTransAppendRedoAction(pTrans, &action); } @@ -190,7 +190,7 @@ class MndTestTrans2 : public ::testing::Test { action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED; mndTransAppendUndoAction(pTrans, &action); } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 7f5b9687fa..d01fdabac8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -153,8 +153,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) if (!pVnode->restored) { vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); - terrno = TSDB_CODE_APP_NOT_READY; - vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_APP_NOT_READY); + terrno = TSDB_CODE_SYN_RESTORING; + vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); continue; @@ -544,21 +544,23 @@ bool vnodeIsRoleLeader(SVnode *pVnode) { } bool vnodeIsLeader(SVnode *pVnode) { + terrno = 0; SSyncState state = syncGetState(pVnode->sync); - if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) { - if (state.state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } - vInfo("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored); + if (terrno != 0) { + vInfo("vgId:%d, vnode is stopping", pVnode->config.vgId); return false; } - if (!pVnode->restored) { - vInfo("vgId:%d, vnode not restored", pVnode->config.vgId); - terrno = TSDB_CODE_APP_NOT_READY; + if (state.state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + vInfo("vgId:%d, vnode not leader, state:%s", pVnode->config.vgId, syncStr(state.state)); + return false; + } + + if (!state.restored || !pVnode->restored) { + terrno = TSDB_CODE_SYN_RESTORING; + vInfo("vgId:%d, vnode not restored:%d:%d", pVnode->config.vgId, state.restored, pVnode->restored); return false; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index d87df2d0f3..3631c4c1b6 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -598,8 +598,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND|| code == TSDB_CODE_APP_IS_STARTING ) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f102095ce2..abfd8043c9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -401,62 +401,62 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) { bool syncNodeIsReadyForRead(SSyncNode* pSyncNode) { if (pSyncNode == NULL) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; sError("sync ready for read error"); return false; } - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + return false; + } + + if (pSyncNode->restoreFinish) { return true; } bool ready = false; - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && !pSyncNode->restoreFinish) { - if (!pSyncNode->pFsm->FpApplyQueueEmptyCb(pSyncNode->pFsm)) { - // apply queue not empty - ready = false; + if (!pSyncNode->pFsm->FpApplyQueueEmptyCb(pSyncNode->pFsm)) { + // apply queue not empty + ready = false; - } else { - if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { - SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SSyncRaftEntry* pEntry = NULL; - SLRUCache* pCache = pSyncNode->pLogStore->pCache; - LRUHandle* h = taosLRUCacheLookup(pCache, &lastIndex, sizeof(lastIndex)); - int32_t code = 0; - if (h) { - pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); - code = 0; + } else { + if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { + SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SSyncRaftEntry* pEntry = NULL; + SLRUCache* pCache = pSyncNode->pLogStore->pCache; + LRUHandle* h = taosLRUCacheLookup(pCache, &lastIndex, sizeof(lastIndex)); + int32_t code = 0; + if (h) { + pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); + code = 0; - pSyncNode->pLogStore->cacheHit++; - sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", lastIndex, pEntry->bytes, pEntry); + pSyncNode->pLogStore->cacheHit++; + sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", lastIndex, pEntry->bytes, pEntry); - } else { - pSyncNode->pLogStore->cacheMiss++; - sNTrace(pSyncNode, "miss cache index:%" PRId64, lastIndex); + } else { + pSyncNode->pLogStore->cacheMiss++; + sNTrace(pSyncNode, "miss cache index:%" PRId64, lastIndex); - code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, lastIndex, &pEntry); + code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, lastIndex, &pEntry); + } + + if (code == 0 && pEntry != NULL) { + if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { + ready = true; } - if (code == 0 && pEntry != NULL) { - if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { - ready = true; - } - - if (h) { - taosLRUCacheRelease(pCache, h, false); - } else { - syncEntryDestroy(pEntry); - } + if (h) { + taosLRUCacheRelease(pCache, h, false); + } else { + syncEntryDestroy(pEntry); } } } } if (!ready) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } + terrno = TSDB_CODE_SYN_RESTORING; } return ready; @@ -549,7 +549,11 @@ SSyncState syncGetState(int64_t rid) { if (pSyncNode != NULL) { state.state = pSyncNode->state; state.restored = pSyncNode->restoreFinish; - state.canRead = syncNodeIsReadyForRead(pSyncNode); + if (pSyncNode->vgId != 1) { + state.canRead = syncNodeIsReadyForRead(pSyncNode); + } else { + state.canRead = state.restored; + } syncNodeRelease(pSyncNode); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3e8c016a74..4677c49d6c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -56,7 +56,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") -TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, "Operation not supported") TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "Out of Memory") @@ -277,6 +276,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog is null") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction") // mnode-mq @@ -308,14 +308,23 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "SMA does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma option") // dnode -TAOS_DEFINE_ERROR(TSDB_CODE_NODE_OFFLINE, "Node is offline") -TAOS_DEFINE_ERROR(TSDB_CODE_NODE_ALREADY_DEPLOYED, "Node already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_NODE_NOT_DEPLOYED, "Node not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_FOUND, "Mnode not found") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_DEPLOYED, "Mnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_QNODE_NOT_FOUND, "Qnode not found") +TAOS_DEFINE_ERROR(TSDB_CODE_QNODE_ALREADY_DEPLOYED, "Qnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_QNODE_NOT_DEPLOYED, "Qnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_FOUND, "Snode not found") +TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Vnode hash mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_NOT_EXISTS, "Table column not exists") From 76fd76cd1ad996255debbe496cf97c29fb173af0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 16:22:06 +0800 Subject: [PATCH 068/120] enh: optimize db vgroup cache and child table meta retrieving --- include/libs/qcom/query.h | 1 + source/libs/catalog/inc/catalogInt.h | 3 +- source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgCache.c | 51 +++++++-- source/libs/catalog/src/ctgUtil.c | 154 ++++----------------------- 5 files changed, 66 insertions(+), 145 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 6319281212..b8d09e2262 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -129,6 +129,7 @@ typedef struct SDBVgInfo { int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo + SArray* vgArray; } SDBVgInfo; typedef struct SUseDbOutput { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d73dcb3b30..085b2c1112 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -764,8 +764,6 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); -int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); - int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); @@ -791,6 +789,7 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog **pHandle); +int ctgVgInfoComp(const void* lp, const void* rp); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f17abaa31f..3a398d1551 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -536,7 +536,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* return TSDB_CODE_SUCCESS; } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue2(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index db3a53aeb0..c319a8350f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -226,6 +226,7 @@ _return: return TSDB_CODE_SUCCESS; } +/* int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; @@ -276,9 +277,9 @@ _return: return TSDB_CODE_SUCCESS; } +*/ - -int32_t ctgAcquireStbMetaFromCache2(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { +int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { SCtgTbCache *pCache = NULL; char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { @@ -437,7 +438,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache2(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); + ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { @@ -503,10 +504,15 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, // PROCESS FOR CHILD TABLE - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + } + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); - ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); @@ -846,6 +852,18 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } + if (dbInfo->vgHash && NULL == dbInfo->vgArray) { + dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(dbInfo->vgArray, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + } + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->pCtg = pCtg; msg->dbId = dbId; @@ -1592,6 +1610,20 @@ void ctgFreeAllInstance(void) { taosHashClear(gCtgMgmt.pCluster); } +int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { + int32_t* key = (int32_t*)lp; + SVgroupInfo* pVg = (SVgroupInfo*)rp; + + if (*key < pVg->vgId) { + return -1; + } else if (*key > pVg->vgId) { + return 1; + } + + return 0; +} + + int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -1973,7 +2005,13 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SVgroupInfo *pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); + ctgDebug("no vgroup %d in db %s vgHash, ignore epset update", msg->vgId, msg->dbFName); + goto _return; + } + + SVgroupInfo *pInfo2 = taosArraySearch(vgInfo->vgArray, &msg->vgId, ctgVgInfoIdComp, TD_EQ); + if (NULL == pInfo2) { + ctgDebug("no vgroup %d in db %s vgArray, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1984,6 +2022,7 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { msg->epSet.numOfEps, pNewEp->fqdn, pNewEp->port, msg->dbFName); pInfo->epSet = msg->epSet; + pInfo2->epSet = msg->epSet; _return: diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index ef7b80a676..6fba94cc8a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -236,10 +236,8 @@ void ctgFreeVgInfo(SDBVgInfo* vgInfo) { return; } - if (vgInfo->vgHash) { - taosHashCleanup(vgInfo->vgHash); - vgInfo->vgHash = NULL; - } + taosHashCleanup(vgInfo->vgHash); + taosArrayDestroy(vgInfo->vgArray); taosMemoryFreeClear(vgInfo); } @@ -837,7 +835,7 @@ _return: CTG_RET(code); } -int ctgVgInfoComp2(const void* lp, const void* rp) { +int ctgVgInfoComp(const void* lp, const void* rp) { SVgroupInfo* pLeft = (SVgroupInfo*)lp; SVgroupInfo* pRight = (SVgroupInfo*)rp; if (pLeft->hashBegin < pRight->hashBegin) { @@ -850,32 +848,6 @@ int ctgVgInfoComp2(const void* lp, const void* rp) { } int32_t ctgHashValueComp(void const* lp, void const* rp) { - uint32_t* key = (uint32_t*)lp; - SVgroupInfo* pVg = *(SVgroupInfo**)rp; - - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; - } - - return 0; -} - -int ctgVgInfoComp(const void* lp, const void* rp) { - SVgroupInfo* pLeft = *(SVgroupInfo**)lp; - SVgroupInfo* pRight = *(SVgroupInfo**)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; - } - - return 0; -} - - -int32_t ctgHashValueComp2(void const* lp, void const* rp) { uint32_t* key = (uint32_t*)lp; SVgroupInfo* pVg = (SVgroupInfo*)rp; @@ -891,7 +863,7 @@ int32_t ctgHashValueComp2(void const* lp, void const* rp) { int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -907,6 +879,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + +/* void* pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; @@ -918,84 +893,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } +*/ if (NULL == vgInfo) { ctgError("0no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - *pVgroup = *vgInfo; - - ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, - vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, - vgInfo->epSet.eps[vgInfo->epSet.inUse].port); - - CTG_RET(code); -} - - -int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { - int32_t code = 0; - - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); - - if (vgNum <= 0) { - ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - - SVgroupInfo* vgInfo = NULL; - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - - uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, - dbInfo->hashPrefix, dbInfo->hashSuffix); - - static SArray* pVgList = NULL; - static bool created = false; - - if (!created) { - ctgDebug("create vg array, %d", taosHashGetSize(dbInfo->vgHash)); - pVgList = taosArrayInit(100, sizeof(SVgroupInfo)); - - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(pVgList, pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(pVgList, ctgVgInfoComp2); - created = true; - } - - vgInfo = taosArraySearch(pVgList, &hashValue, ctgHashValueComp2, TD_EQ); - - if (NULL == vgInfo) { - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } - - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } - - if (vgInfo) { - taosArrayDestroy(pVgList); - created = false; - ctgDebug("need to re-create vg array, %d", taosHashGetSize(dbInfo->vgHash)); - } - } - - if (NULL == vgInfo) { - ctgError("1no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosHashGetSize(dbInfo->vgHash)); + taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1013,7 +915,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -1023,20 +925,13 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t tbNum = taosArrayGetSize(pNames); if (1 == vgNum) { - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - if (NULL == pIter) { - ctgError("empty vgHash, db:%s, vgroup number:%d", dbFName, vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { - taosHashCancelIterate(dbInfo->vgHash, pIter); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - *vgInfo = *(SVgroupInfo*)pIter; + *vgInfo = *(SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); @@ -1051,19 +946,9 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* } } - taosHashCancelIterate(dbInfo->vgHash, pIter); return TSDB_CODE_SUCCESS; } - SArray* pVgList = taosArrayInit(vgNum, POINTER_BYTES); - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(pVgList, &pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(pVgList, ctgVgInfoComp); - char tbFullName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); @@ -1079,20 +964,15 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); - SVgroupInfo** p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); - - if (NULL == p) { + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + if (NULL == vgInfo) { ctgError("2no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, - taosHashGetSize(dbInfo->vgHash)); - taosArrayDestroy(pVgList); + taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - vgInfo = *p; - SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pNewVg) { - taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1112,8 +992,6 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* } } - taosArrayDestroy(pVgList); - CTG_RET(code); } @@ -1190,6 +1068,10 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { pIter = taosHashIterate(src->vgHash, pIter); } + if (src->vgArray) { + (*dst)->vgArray = taosArrayDup(src->vgArray, NULL); + } + return TSDB_CODE_SUCCESS; } From 7b18f0a8422b678e8f412ae9bebcc3a51f28e50d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 16:36:56 +0800 Subject: [PATCH 069/120] fix: fix compile issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 6fba94cc8a..120e261c32 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -896,8 +896,8 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName */ if (NULL == vgInfo) { - ctgError("0no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosArrayGetSize(dbInfo->vgArray)); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } From 3c1e52eb91783c2717e54af453b399cbf2c50c5c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 16:40:09 +0800 Subject: [PATCH 070/120] fix: fix size_t compile issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 120e261c32..17ee8bb68e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -966,8 +966,8 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); if (NULL == vgInfo) { - ctgError("2no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, - taosArrayGetSize(dbInfo->vgArray)); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } From 39482b24db8660c685f6caf5a848401f647f90f7 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 16:42:38 +0800 Subject: [PATCH 071/120] feat(shell): supported affected rows int64_t --- include/client/taos.h | 1 + source/client/inc/clientInt.h | 4 ++-- source/client/src/clientMain.c | 12 ++++++++++++ source/libs/executor/src/dataDispatcher.c | 2 +- tools/shell/src/shellEngine.c | 16 ++++++++++++++-- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 2940f1dfd0..69774b750f 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -185,6 +185,7 @@ DLL_EXPORT void taos_kill_query(TAOS *taos); DLL_EXPORT int taos_field_count(TAOS_RES *res); DLL_EXPORT int taos_num_fields(TAOS_RES *res); DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index aae20c587d..6c2a7d75a0 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -171,9 +171,9 @@ typedef struct SReqResultInfo { char** convertBuf; TAOS_ROW row; SResultColumn* pCol; - uint32_t numOfRows; + uint64_t numOfRows; uint64_t totalRows; - uint32_t current; + uint64_t current; bool localResultFetched; bool completed; int32_t precision; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 976d1dd1b0..fecb94d02c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -438,11 +438,23 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } +// return int32_t int taos_affected_rows(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return 0; } + SRequestObj *pRequest = (SRequestObj *)res; + SReqResultInfo *pResInfo = &pRequest->body.resInfo; + return (int)pResInfo->numOfRows; +} + +// return int64_t +int64_t taos_affected_rows(TAOS_RES *res) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { + return 0; + } + SRequestObj *pRequest = (SRequestObj *)res; SReqResultInfo *pResInfo = &pRequest->body.resInfo; return pResInfo->numOfRows; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d4248fc420..78f6155cf7 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -189,7 +189,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); - pOutput->numOfRows = pEntry->numOfRows; + pOutput->fr = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 118a6caf7a..362a9b3c80 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -214,6 +214,18 @@ void shellRunSingleCommandImp(char *command) { return; } + // pre string + char * pre = "Query OK"; + if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Deleted OK"; + } else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Inserted OK"; + } else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Created OK"; + } else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Droped OK"; + } + TAOS_FIELD *pFields = taos_fetch_fields(pSql); if (pFields != NULL) { // select and show kinds of commands int32_t error_no = 0; @@ -229,10 +241,10 @@ void shellRunSingleCommandImp(char *command) { } taos_free_result(pSql); } else { - int32_t num_rows_affacted = taos_affected_rows(pSql); + int64_t num_rows_affacted = taos_affected_rows64(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); From 4b13ae819d0aca3ae43f441834f09e5e89a6a53b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 16:45:41 +0800 Subject: [PATCH 072/120] enh: adjust error codes --- include/libs/qcom/query.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 4521b32409..e2e9da6624 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -264,12 +264,12 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later -#define NEED_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ - SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ - (_code) == TSDB_CODE_APP_IS_STARTING) +#define NEED_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ + (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ + (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ @@ -279,7 +279,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_MNODE_NOT_FOUND || \ SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_SYN_RESTORING || \ - (_code) == TSDB_CODE_APP_IS_STARTING) + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define REQUEST_TOTAL_EXEC_TIMES 2 From 99968ccc4fb7db9d88be18f78e28cfbb0ce010f7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 16:46:43 +0800 Subject: [PATCH 073/120] enh: adjust error code --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/libs/function/src/udfd.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5926936de7..6e7d869eed 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -245,7 +245,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcRe static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || TSDB_CODE_APP_IS_STOPPING) { + code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 3631c4c1b6..ec54f8ff2d 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -599,7 +599,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND|| code == TSDB_CODE_APP_IS_STARTING ) { + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND|| code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } From 16cd6ede5f870850909a960b44dcd6f0b03b59d5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 16:49:10 +0800 Subject: [PATCH 074/120] enh: adjust error codes --- include/util/taoserror.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9e4ede9afb..114cc45a59 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -340,14 +340,14 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) // 2.x // #define TSDB_CODE_DND_VNODE_OPEN_FAILED TAOS_DEF_ERROR_CODE(0, 0x0407) // 2.x #define TSDB_CODE_DNODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408) -#define TSDB_CODE_MNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0409) -#define TSDB_CODE_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040A) +#define TSDB_CODE_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0409) +#define TSDB_CODE_MNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040A) #define TSDB_CODE_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040B) -#define TSDB_CODE_QNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040C) -#define TSDB_CODE_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040D) +#define TSDB_CODE_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040C) +#define TSDB_CODE_QNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040D) #define TSDB_CODE_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040E) -#define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040F) -#define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0410) +#define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040F) +#define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0410) #define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) // vnode From 878f2a2c19c8d9f6859546651c1e66d18a245781 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:01:40 +0800 Subject: [PATCH 075/120] fix(shell): reset dataDispatcher mistake modify --- source/libs/executor/src/dataDispatcher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 78f6155cf7..d4248fc420 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -189,7 +189,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); - pOutput->fr = pEntry->numOfRows; + pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; From 2aa65c4b9c6d7ed7d1d09cf41640cfc9b59f3d67 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:09:04 +0800 Subject: [PATCH 076/120] feat(shell): affected rows int64 --- source/client/src/clientImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 846accf51b..77699e89c2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -317,7 +317,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); } @@ -1527,7 +1527,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) return NULL; } - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; From 0f0e575481febaae34d084b42378342942dff712 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:14:49 +0800 Subject: [PATCH 077/120] feat(shell): affected rows int64 --- source/client/src/clientMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index fecb94d02c..8020134c52 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -450,7 +450,7 @@ int taos_affected_rows(TAOS_RES *res) { } // return int64_t -int64_t taos_affected_rows(TAOS_RES *res) { +int64_t taos_affected_rows64(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return 0; } @@ -971,7 +971,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); From 83a4f76ecea33eeaaa04db2d03dbb08455aeb917 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:15:51 +0800 Subject: [PATCH 078/120] feat(shell): affected rows int64 --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 362a9b3c80..14757247b7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -244,7 +244,7 @@ void shellRunSingleCommandImp(char *command) { int64_t num_rows_affacted = taos_affected_rows64(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", pre, num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); From 867f7abeea3099dcf3ea61fb64a8dabb9c53f049 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 17:44:39 +0800 Subject: [PATCH 079/120] test: add asan case --- tests/parallel_test/cases.task | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e514b069a8..13bfe4c981 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -623,12 +623,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -,,,system-test,python3 ./test.py -f 2-query/stablity.py -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py @@ -788,12 +788,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2 @@ -881,12 +881,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3 @@ -974,10 +974,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 From 27ed740eb56317a5bcb5dc4328fecca8179a51b4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 17:52:49 +0800 Subject: [PATCH 080/120] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 89 +++++++++++++------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 859ae026a2..ac003783ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -81,15 +81,11 @@ typedef struct SIOCostSummary { double createScanInfoList; } SIOCostSummary; -typedef struct SColInfo { - int16_t colId; - int16_t slotId; -} SColInfo; - typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; - SColInfo* colInfo; // column ids for loading file block data + int16_t* colId; + int16_t* slotId; int32_t numOfCols; char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. bool smaValid; // the sma on all queried columns are activated @@ -222,16 +218,17 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES)); - if (pSupInfo->colInfo == NULL) { - taosMemoryFree(pSupInfo->colInfo); + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES)); + if (pSupInfo->colId == NULL) { + taosMemoryFree(pSupInfo->colId); return TSDB_CODE_OUT_OF_MEMORY; } - pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols)); + pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { - pSupInfo->colInfo[i].colId = pCols[i].colId; - pSupInfo->colInfo[i].slotId = pSlotIdList[i]; + pSupInfo->colId[i] = pCols[i].colId; + pSupInfo->slotId[i] = pSlotIdList[i]; if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); @@ -248,7 +245,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; - if (pTCol->colId == pSupInfo->colInfo[j].colId) { + if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; return; @@ -256,7 +253,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) i += 1; j += 1; - } else if (pTCol->colId < pSupInfo->colInfo[j].colId) { + } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { @@ -458,7 +455,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb if (pLReader->pInfo == NULL) { // here we ignore the first column, which is always be the primary timestamp column pLReader->pInfo = - tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); + tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1); if (pLReader->pInfo == NULL) { tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); return terrno; @@ -1094,8 +1091,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); - if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1106,10 +1103,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn rowIndex = 0; SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pSupInfo->colInfo[i].colId) { + if (pData->cid < pSupInfo->colId[i]) { colIndex += 1; - } else if (pData->cid == pSupInfo->colInfo[i].colId) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + } else if (pData->cid == pSupInfo->colId[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); @@ -1127,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1135,7 +1132,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1173,7 +1170,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); TABLEID tid = {.suid = pReader->suid, .uid = uid}; int32_t code = - tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); + tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1632,7 +1629,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2504,7 +2501,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId); + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -3553,24 +3550,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { - col_id_t colId = pSupInfo->colInfo[i].colId; + col_id_t colId = pSupInfo->colId[i]; if (colId == pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; @@ -3581,7 +3578,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3597,8 +3594,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } @@ -3609,13 +3606,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols && j < numOfInputCols) { SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - if (pData->cid < pSupInfo->colInfo[i].colId) { + if (pData->cid < pSupInfo->colId[i]) { j += 1; continue; } - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); - if (pData->cid == pSupInfo->colInfo[i].colId) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pData->cid == pSupInfo->colId[i]) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3628,7 +3625,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pCol, outputRowIndex); i += 1; } @@ -3897,7 +3894,7 @@ void tsdbReaderClose(STsdbReader* pReader) { pReader->pResBlock = blockDataDestroy(pReader->pResBlock); } - taosMemoryFree(pSupInfo->colInfo); + taosMemoryFree(pSupInfo->colId); tBlockDataDestroy(&pReader->status.fileBlockData, true); cleanupDataBlockIterator(&pReader->status.blockIter); @@ -4099,21 +4096,21 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockS while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colInfo[j].colId) { - pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg; + if (pAgg->colId == pSup->colId[j]) { + pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; i += 1; j += 1; - } else if (pAgg->colId < pSup->colInfo[j].colId) { + } else if (pAgg->colId < pSup->colId[j]) { i += 1; - } else if (pSup->colInfo[j].colId < pAgg->colId) { - if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg; + } else if (pSup->colId[j] < pAgg->colId) { + if (pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { + pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg; } else { // all date in this block are null - SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow}; + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = pBlock->nRow}; taosArrayPush(pSup->pColAgg, &nullColAgg); - pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg); + pResBlock->pBlockAgg[pSup->slotId[j]] = taosArrayGetLast(pSup->pColAgg); } j += 1; } From 55fb892e70bd6631196f1078aea31b27f25f7abc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 2 Dec 2022 15:50:53 +0800 Subject: [PATCH 081/120] fix(tsdb/cache): use lru erase to invalidate cache entries --- source/dnode/vnode/src/tsdb/tsdbCache.c | 125 +++++++----------------- 1 file changed, 35 insertions(+), 90 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d71eb33951..7309ea3407 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -228,23 +228,23 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST invalidate = true; break; - } - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); + } else { // new inserting key is greater than cached, update cached entry + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - taosArraySet(pLast, iCol, &lastCol); + taosArraySet(pLast, iCol, &lastCol); + } } } } @@ -253,65 +253,10 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); - /* - cacheRow = (STSRow *)taosLRUCacheValue(pCache, h); - if (row->ts >= cacheRow->ts) { - if (row->ts == cacheRow->ts) { - STSRow *mergedRow = NULL; - SRowMerger merger = {0}; - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - - tRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema); - - tRowMerge(&merger, &tsdbRowFromTSRow(1, row)); - - tRowMergerGetRow(&merger, &mergedRow); - tRowMergerClear(&merger); - - taosMemoryFreeClear(pTSchema); - - row = mergedRow; - dup = false; - } - - if (TD_ROW_LEN(row) <= TD_ROW_LEN(cacheRow)) { - tdRowCpy(cacheRow, row); - if (!dup) { - taosMemoryFree(row); - } - - taosLRUCacheRelease(pCache, h, false); - } else { - taosLRUCacheRelease(pCache, h, true); - // tsdbCacheDeleteLastrow(pCache, uid, TSKEY_MAX); - if (dup) { - cacheRow = tdRowDup(row); - } else { - cacheRow = row; - } - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, - TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - // tsdbCacheInsertLastrow(pCache, uid, row, dup); - } - }*/ - } /*else { - if (dup) { - cacheRow = tdRowDup(row); - } else { - cacheRow = row; + if (invalidate) { + taosLRUCacheErase(pCache, key, keyLen); } - - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - }*/ + } return code; } @@ -349,28 +294,28 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal colVal = {0}; tTSRowGetVal(row, pTSchema, iCol, &colVal); - if (!COL_VAL_IS_VALUE(&colVal)) { + if (COL_VAL_IS_VALUE(&colVal)) { if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; break; - } - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); + } else { + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - taosArraySet(pLast, iCol, &lastCol); + taosArraySet(pLast, iCol, &lastCol); + } } } } @@ -379,9 +324,9 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); - - // clear last cache anyway, lazy load when get last lookup - // taosLRUCacheRelease(pCache, h, true); + if (invalidate) { + taosLRUCacheErase(pCache, key, keyLen); + } } return code; From b08ee99c4c746f1c2c991fc54819bf4d017e1f7e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 18:09:38 +0800 Subject: [PATCH 082/120] fix: vgroup array not made in some cases --- include/libs/qcom/query.h | 1 + source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 14 ++++----- source/libs/catalog/src/ctgCache.c | 28 ++++++++--------- source/libs/catalog/src/ctgUtil.c | 45 ++++++++++++++++++---------- source/libs/qcom/src/queryUtil.c | 16 ++++++++-- 6 files changed, 62 insertions(+), 44 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b8d09e2262..268649524f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -239,6 +239,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, void* (*mallocFp)(int64_t)); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 085b2c1112..ba99906589 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -762,7 +762,6 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList); void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); -void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); @@ -790,6 +789,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog **pHandle); int ctgVgInfoComp(const void* lp, const void* rp); +int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d1551..2b1bdeadf8 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -505,8 +505,7 @@ _return: taosMemoryFreeClear(tbMeta); if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } if (vgList) { @@ -546,8 +545,7 @@ _return: } if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } CTG_RET(code); @@ -778,8 +776,7 @@ _return: } if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } CTG_API_LEAVE(code); @@ -836,8 +833,7 @@ _return: ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } else if (dbInfo) { - taosHashCleanup(dbInfo->vgHash); - taosMemoryFreeClear(dbInfo); + freeVgInfo(dbInfo); } CTG_API_LEAVE(code); @@ -849,7 +845,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t code = 0; if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index c319a8350f..4626d4527d 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -843,7 +843,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); taosMemoryFree(op); - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -852,16 +852,12 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } - if (dbInfo->vgHash && NULL == dbInfo->vgArray) { - dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); - - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(dbInfo->vgArray, pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + code = ctgMakeVgArray(dbInfo); + if (code) { + taosMemoryFree(op); + taosMemoryFree(msg); + freeVgInfo(dbInfo); + CTG_ERR_RET(code); } tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); @@ -877,7 +873,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId _return: - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_RET(code); } @@ -1675,7 +1671,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { goto _return; } - ctgFreeVgInfo(vgInfo); + freeVgInfo(vgInfo); } vgCache->vgInfo = dbInfo; @@ -1696,7 +1692,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { _return: - ctgFreeVgInfo(msg->dbInfo); + freeVgInfo(msg->dbInfo); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1749,7 +1745,7 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgCache.vgInfo); + freeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); @@ -2139,7 +2135,7 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) { switch (op->opId) { case CTG_OP_UPDATE_VGROUP: { SCtgUpdateVgMsg *msg = op->data; - ctgFreeVgInfo(msg->dbInfo); + freeVgInfo(msg->dbInfo); taosMemoryFreeClear(op->data); break; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 17ee8bb68e..67db0c79d6 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -231,18 +231,7 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) { CTG_CACHE_STAT_DEC(numOfTbl, tblNum); } -void ctgFreeVgInfo(SDBVgInfo* vgInfo) { - if (NULL == vgInfo) { - return; - } - - taosHashCleanup(vgInfo->vgHash); - taosArrayDestroy(vgInfo->vgArray); - - taosMemoryFreeClear(vgInfo); -} - -void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { ctgFreeVgInfo(dbCache->vgCache.vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); } void ctgFreeDbCache(SCtgDBCache* dbCache) { if (NULL == dbCache) { @@ -364,8 +353,7 @@ void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) { } if (pOutput->dbVgroup) { - taosHashCleanup(pOutput->dbVgroup->vgHash); - taosMemoryFreeClear(pOutput->dbVgroup); + freeVgInfo(pOutput->dbVgroup); } taosMemoryFree(pOutput); @@ -571,8 +559,7 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void** pRes) { case CTG_TASK_GET_DB_VGROUP: { if (*pRes) { SDBVgInfo* pInfo = (SDBVgInfo*)*pRes; - taosHashCleanup(pInfo->vgHash); - taosMemoryFreeClear(*pRes); + freeVgInfo(pInfo); } break; } @@ -1035,7 +1022,33 @@ int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { } } +int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { + if (NULL == dbInfo) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgHash && NULL == dbInfo->vgArray) { + dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); + if (NULL == dbInfo->vgArray) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(dbInfo->vgArray, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { + CTG_ERR_RET(ctgMakeVgArray(src)); + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); if (NULL == *dst) { qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 6eadf80e3d..96e02c3643 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -447,6 +447,19 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } + +void freeVgInfo(SDBVgInfo* vgInfo) { + if (NULL == vgInfo) { + return; + } + + taosHashCleanup(vgInfo->vgHash); + taosArrayDestroy(vgInfo->vgArray); + + taosMemoryFreeClear(vgInfo); +} + + int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -475,8 +488,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) { qError("taosHashPut failed, vgId:%d", vgInfo->vgId); taosHashCancelIterate(pSrc->vgHash, pIter); - taosHashCleanup((*pDst)->vgHash); - taosMemoryFreeClear(*pDst); + freeVgInfo(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } From 693a602c62bc888a2d436404df49fc2172952cc9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 18:27:18 +0800 Subject: [PATCH 083/120] feat(api): modify prompt text --- tools/shell/src/shellEngine.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 14757247b7..5be46fe54a 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -217,13 +217,13 @@ void shellRunSingleCommandImp(char *command) { // pre string char * pre = "Query OK"; if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Deleted OK"; + pre = "Delete OK"; } else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Inserted OK"; + pre = "Insert OK"; } else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Created OK"; + pre = "Create OK"; } else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Droped OK"; + pre = "Drop OK"; } TAOS_FIELD *pFields = taos_fetch_fields(pSql); From 61cd7539475df28316708933cc46417a381699ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 18:59:03 +0800 Subject: [PATCH 084/120] enh: remove sync batch propose in vnode --- source/dnode/vnode/src/vnd/vnodeSync.c | 74 ++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d01fdabac8..ee95e5b7d1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vnd.h" -#define BATCH_DISABLE 1 +#define BATCH_ENABLE 0 static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || @@ -87,7 +87,7 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { } static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { - if (code == TSDB_CODE_SYN_NOT_LEADER) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { vnodeRedirectRpcMsg(pVnode, pMsg); } else { const STraceId *trace = &pMsg->info.traceId; @@ -99,15 +99,12 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) } } +#if BATCH_ENABLE + static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) { if (*arrSize <= 0) return; -#if BATCH_DISABLE - int32_t code = syncPropose(pVnode->sync, pMsgArr[0], pIsWeakArr[0]); -#else int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize); -#endif - if (code > 0) { for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleWriteMsg(pVnode, pMsgArr[i]); @@ -177,7 +174,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - if (isBlock || BATCH_DISABLE) { + if (isBlock) { vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos); } @@ -185,7 +182,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) pIsWeakArr[arrayPos] = isWeak; arrayPos++; - if (isBlock || msg == numOfMsgs - 1 || BATCH_DISABLE) { + if (isBlock || msg == numOfMsgs - 1) { vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos); } } @@ -194,6 +191,65 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) taosMemoryFree(pIsWeakArr); } +#else + +static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { + int32_t code = syncPropose(pVnode->sync, pMsg, isWeak); + if (code > 0) { + vnodeHandleWriteMsg(pVnode, pMsg); + } else if (code == 0) { + vnodeWaitBlockMsg(pVnode, pMsg); + } else { + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + } + + return code; +} + +void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnode *pVnode = pInfo->ahandle; + int32_t vgId = pVnode->config.vgId; + int32_t code = 0; + SRpcMsg *pMsg = NULL; + vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs); + + for (int32_t msg = 0; msg < numOfMsgs; msg++) { + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + bool isWeak = vnodeIsMsgWeak(pMsg->msgType); + + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p get from vnode-write queue, weak:%d block:%d msg:%d:%d, handle:%p", vgId, pMsg, isWeak, + vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); + + if (!pVnode->restored) { + vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); + vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = vnodePreProcessWriteMsg(pVnode, pMsg); + if (code != 0) { + vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = vnodeProposeMsg(pVnode, pMsg, isWeak); + + vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } +} + +#endif + void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnode *pVnode = pInfo->ahandle; int32_t vgId = pVnode->config.vgId; From cafcec2571a409b6f8cd2adb0d74a2f86a3946be Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 20:09:44 +0800 Subject: [PATCH 085/120] fix: make vgroup array when needed --- source/libs/catalog/src/ctgUtil.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 67db0c79d6..802ecde63e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -849,6 +849,7 @@ int32_t ctgHashValueComp(void const* lp, void const* rp) { int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; @@ -902,6 +903,9 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; + + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); From 9fb4fa0a7f8443848b0171158d994fe164e10fa8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 20:48:39 +0800 Subject: [PATCH 086/120] feat(api): add int64 affected rows modify tmsg.h struct --- include/common/tmsg.h | 4 ++-- source/client/inc/clientInt.h | 2 +- source/client/src/clientImpl.c | 2 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/exchangeoperator.c | 10 +++++----- source/libs/scheduler/inc/schInt.h | 2 +- source/libs/scheduler/src/schJob.c | 4 ++-- source/libs/scheduler/src/schRemote.c | 8 ++++---- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0a082a37e4..9ad2e4375b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1403,8 +1403,8 @@ typedef struct { int8_t streamBlockType; int32_t compLen; int32_t numOfBlocks; - int32_t numOfRows; - int32_t numOfCols; + int64_t numOfRows; // from int32_t change to int64_t + int64_t numOfCols; int64_t skey; int64_t ekey; int64_t version; // for stream diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6c2a7d75a0..ea76f726ea 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -171,7 +171,7 @@ typedef struct SReqResultInfo { char** convertBuf; TAOS_ROW row; SResultColumn* pCol; - uint64_t numOfRows; + uint64_t numOfRows; // from int32_t change to int64_t uint64_t totalRows; uint64_t current; bool localResultFetched; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 77699e89c2..6444150263 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1941,7 +1941,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->numOfRows = htonll(pRsp->numOfRows); pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7100be58e3..60f8171bac 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -684,7 +684,7 @@ void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pC int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 963a273290..03e9e411bb 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -115,14 +115,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -367,14 +367,14 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pSourceDataInfo->pRsp = pMsg->pData; SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->numOfRows = htonll(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); @@ -472,7 +472,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return TSDB_CODE_SUCCESS; } -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator) { pInfo->totalRows += numOfRows; pInfo->totalSize += dataLen; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index dfc48e7d9f..77aafa9a27 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -297,7 +297,7 @@ typedef struct SSchJob { SExecResult execRes; void *fetchRes; // TODO free it or not bool fetched; - int32_t resNumOfRows; + int64_t resNumOfRows; // from int32_t to int64_t SSchResInfo userRes; char *sql; SQueryProfileSummary summary; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 0eb29a3667..37a3cc9286 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -412,7 +412,7 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } - SCH_JOB_DLOG("fetch done, totalRows:%d", pJob->resNumOfRows); + SCH_JOB_DLOG("fetch done, totalRows:%" PRId64, pJob->resNumOfRows); _return: @@ -528,7 +528,7 @@ void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); - atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); + atomic_store_64(&pJob->resNumOfRows, htonll(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a6d7df349..101fdfe052 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -108,7 +108,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); + atomic_add_fetch_64(&pJob->resNumOfRows, htonll(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -279,7 +279,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa } } - atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); SCH_LOCK(SCH_WRITE, &pJob->resLock); @@ -317,7 +317,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa tDecodeSVDeleteRsp(&coder, &rsp); tDecoderClear(&coder); - atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -344,7 +344,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); From 855b0c8cbe5b4ff45effd78a7b348367715c95be Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 20:57:46 +0800 Subject: [PATCH 087/120] feat(api): add int64 affected rows modify tmsg.h struct --- source/client/src/clientImpl.c | 2 +- source/libs/executor/src/exchangeoperator.c | 6 +++--- source/libs/scheduler/src/schJob.c | 2 +- source/libs/scheduler/src/schRemote.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6444150263..f823b527ea 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1941,7 +1941,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonll(pRsp->numOfRows); + pResultInfo->numOfRows = htobe64(pRsp->numOfRows); pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 03e9e411bb..dfbfe62e13 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -367,7 +367,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pSourceDataInfo->pRsp = pMsg->pData; SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonll(pRsp->numOfRows); + pRsp->numOfRows = htobe64(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); @@ -655,7 +655,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, @@ -664,7 +664,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 37a3cc9286..ea49b0c4e0 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -528,7 +528,7 @@ void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); - atomic_store_64(&pJob->resNumOfRows, htonll(pRsp->numOfRows)); + atomic_store_64(&pJob->resNumOfRows, htobe64(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 101fdfe052..76ac0a2e9f 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -108,7 +108,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_64(&pJob->resNumOfRows, htonll(rsp->numOfRows)); + atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); From 524de026a553a83500d288b27f3a397561d0bd51 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 22:24:35 +0800 Subject: [PATCH 088/120] enh: remove TSDB_CODE_RPC_REDIRECT --- include/common/tmsgcb.h | 3 --- include/libs/qcom/query.h | 18 ++++++++--------- include/util/taoserror.h | 4 ++-- source/client/src/clientMain.c | 7 ++----- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 20 +------------------ source/dnode/mnode/impl/src/mndTrans.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSync.c | 12 ++++++++++- source/libs/function/src/udfd.c | 7 ++++--- source/libs/qworker/src/qwDbg.c | 6 +++--- source/libs/transport/src/tmsgcb.c | 2 -- source/libs/transport/src/transCli.c | 2 +- source/util/src/terror.c | 4 +--- 13 files changed, 38 insertions(+), 55 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index b5b997dac0..32d00bb422 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -43,7 +43,6 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); -typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); @@ -55,7 +54,6 @@ typedef struct { GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; - SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; @@ -66,7 +64,6 @@ int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index e2e9da6624..cd61c29559 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -265,21 +265,21 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later #define NEED_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ + (_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ - (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ + (_code) == TSDB_CODE_APP_IS_STOPPING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ (_type) == TDMT_MND_DROP_STB) -#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_MNODE_NOT_FOUND || \ - SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ - SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_SYN_RESTORING || \ - (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) +#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ + (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ + (_code) == TSDB_CODE_APP_IS_STOPPING) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 114cc45a59..d9e9f5a4fd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -42,8 +42,8 @@ int32_t* taosGetErrno(); // rpc // #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x // #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x -#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) -#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) // #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x // #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x // #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 976d1dd1b0..8ee774ceee 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,6 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,12 +153,11 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -171,8 +169,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) - : (const char *)tstrerror(pRequest->code); + return (const char *)tstrerror(pRequest->code); } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 212de0bfb4..857fbcbce5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -61,7 +61,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { pMsg->info.rsp = NULL; } - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { mndPostProcessQueryMsg(pMsg); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 6e7d869eed..a3430d0bde 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -33,23 +33,6 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { } } -static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { - pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType}; - int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet); - - rsp.pCont = rpcMallocCont(contLen); - if (rsp.pCont == NULL) { - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet); - rsp.contLen = contLen; - } - dmSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; -} - int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; @@ -243,7 +226,7 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); } static bool rpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || @@ -334,7 +317,6 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { .clientRpc = pDnode->trans.clientRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, - .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartup, diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 38bfe7f6cb..4b11884050 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -945,7 +945,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (i != 0 && code == 0) { - code = TSDB_CODE_RPC_REDIRECT; + code = TSDB_CODE_MNODE_NOT_FOUND; } mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), pInfo->ahandle); @@ -1042,8 +1042,8 @@ static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pA pAction->rawWritten = 0; pAction->msgSent = 0; pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + if (pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || + pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; mInfo("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->epSet.inUse); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ee95e5b7d1..ed612ff6a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -67,7 +67,17 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { pMsg->info.hasEpSet = 1; SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; - tmsgSendRedirectRsp(&rsp, &newEpSet); + int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet); + + rsp.pCont = rpcMallocCont(contLen); + if (rsp.pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSEpSet(rsp.pCont, contLen, &newEpSet); + rsp.contLen = contLen; + } + + tmsgSendRsp(&rsp); } static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ec54f8ff2d..9d249e0831 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -598,9 +598,10 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND|| code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } return true; diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 4c4a41df82..a122508f98 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -214,20 +214,20 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { epSet.eps[2].port = 7300; ctx->phase = QW_PHASE_POST_QUERY; - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet); *rsped = true; return; } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); *rsped = true; return; } if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) { - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); *rsped = true; return; } diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 559715084c..95bc532994 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -54,8 +54,6 @@ void tmsgSendRsp(SRpcMsg* pMsg) { #endif } -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); } - void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLinkArgFp)(pMsg); } void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a9afbd7ba8..d588e2db1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1519,7 +1519,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4082e8985a..2110c0b3aa 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -46,8 +46,6 @@ STaosError errors[] = { #endif // rpc -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Database not ready, need retry") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") @@ -320,7 +318,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deploye TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") From f3e980de149c5053bffa7d861596d4c4d506631b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 22:45:11 +0800 Subject: [PATCH 089/120] enh: remove TSDB_CODE_RPC_REDIRECT --- include/libs/qcom/query.h | 12 +++++------- include/util/taoserror.h | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 6 +++--- source/dnode/vnode/src/inc/vnd.h | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSync.c | 8 +++++--- source/libs/transport/src/transCli.c | 4 +++- 7 files changed, 21 insertions(+), 19 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cd61c29559..97ce5ae032 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -261,25 +261,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) #define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED) -#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later #define NEED_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ - (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ (_type) == TDMT_MND_DROP_STB) #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + ((_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ - (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d9e9f5a4fd..2baf4d14e2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -43,7 +43,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x // #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x // #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x -// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //2.x // #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x // #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x // #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index a3430d0bde..12aba130d5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -226,9 +226,9 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); } static bool rpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e2b7327e8f..15e64ffaed 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -100,7 +100,7 @@ int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); -void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); +void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d63c45fc0e..fc6b126cc5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -344,7 +344,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); // if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } @@ -367,12 +367,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { pMsg->msgType == TDMT_VND_BATCH_META) && !syncIsReadyForRead(pVnode->sync)) { // !vnodeIsLeader(pVnode)) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ed612ff6a6..a155e8da86 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -53,7 +53,7 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { } } -void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { +void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { SEpSet newEpSet = {0}; syncGetRetryEpSet(pVnode->sync, &newEpSet); @@ -66,7 +66,9 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { } pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; + if (code == 0) code = TSDB_CODE_SYN_NOT_LEADER; + + SRpcMsg rsp = {.code = code, .info = pMsg->info, .msgType = pMsg->msgType + 1}; int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet); rsp.pCont = rpcMallocCont(contLen); @@ -98,7 +100,7 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, code); } else { const STraceId *trace = &pMsg->info.traceId; vGError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", pVnode->config.vgId, pMsg, tstrerror(code), code); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d588e2db1c..7339d487d1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1519,7 +1519,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 5e77b3eca3025815967c44111e959d4c5efb64c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 23:10:03 +0800 Subject: [PATCH 090/120] refactor: add function for extract the required buffer during query. --- source/libs/executor/inc/executorimpl.h | 21 +++--- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/exchangeoperator.c | 2 +- source/libs/executor/src/executor.c | 17 +++-- source/libs/executor/src/executorimpl.c | 73 ++++++------------- source/libs/executor/src/filloperator.c | 4 +- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/joinoperator.c | 2 +- source/libs/executor/src/projectoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 18 ++--- source/libs/executor/src/sortoperator.c | 8 +- source/libs/executor/src/sysscanoperator.c | 4 +- source/libs/executor/src/timesliceoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 38 ++++------ .../libs/function/src/detail/tavgfunction.c | 23 ++---- 15 files changed, 93 insertions(+), 133 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ec2fadef28..9d32bf830e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -46,7 +46,6 @@ extern "C" { typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) @@ -109,6 +108,7 @@ typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param); typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id @@ -170,8 +170,9 @@ struct SExecTaskInfo { STaskCostInfo cost; int64_t owner; // if it is in execution int32_t code; + int32_t qbufQuota; // total available buffer (in KB) during execution query - int64_t version; // used for stream to record wal version + int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; STableListInfo* pTableInfoList; // this is a table list @@ -198,6 +199,7 @@ typedef struct SOperatorFpSet { __optr_fn_t getNextFn; __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP __optr_close_fn_t closeFn; + __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator __optr_encode_fn_t encodeResultRow; __optr_decode_fn_t decodeResultRow; __optr_explain_fn_t getExplainFn; @@ -486,12 +488,10 @@ typedef struct STableCountScanSupp { int16_t dbNameSlotId; int16_t stbNameSlotId; int16_t tbCountSlotId; - - bool groupByDbName; - bool groupByStbName; + bool groupByDbName; + bool groupByStbName; char dbNameFilter[TSDB_DB_NAME_LEN]; char stbNameFilter[TSDB_TABLE_NAME_LEN]; - } STableCountScanSupp; typedef struct STableCountScanOperatorInfo { @@ -671,13 +671,14 @@ typedef struct SStreamFillOperatorInfo { #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_explain_fn_t explain); -int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); +int32_t optrDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); void setOperatorCompleted(SOperatorInfo* pOperator); void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, SExecTaskInfo* pTaskInfo); void destroyOperatorInfo(SOperatorInfo* pOperator); +int32_t optrDefaultBufFn(SOperatorInfo* pOperator); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); @@ -740,6 +741,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); @@ -812,8 +815,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -int32_t getMaximumIdleDurationSec(); - STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index c432f3c01c..672bb09b14 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -117,7 +117,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, NULL); + createOperatorFpSet(optrDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, optrDefaultBufFn, NULL); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 546f5cc420..76e27f12fc 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -307,7 +307,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _error: diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index aedab23f74..60080d204b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -503,7 +503,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo } if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); + pTaskInfo->cost.start = taosGetTimestampUs(); } if (isTaskKilled(pTaskInfo)) { @@ -597,7 +597,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { } if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); + pTaskInfo->cost.start = taosGetTimestampUs(); } if (isTaskKilled(pTaskInfo)) { @@ -706,15 +706,20 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; + int64_t idleTime = pSummary->start - pSummary->created; SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " + "%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " + "createGroupIdMap:%.2f ms, total blocks:%d, " "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, - pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, - pRecorder->totalCheckedRows); + GET_TASKID(pTaskInfo), idleTime / 1000.0, pSummary->elapsedTime / 1000.0, pSummary->extractListTime, + pSummary->groupIdMapTime, pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, + pRecorder->totalRows, pRecorder->totalCheckedRows); + } else { + qDebug("%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), idleTime / 1000.0, + pSummary->elapsedTime / 1000.0); } } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fb705dacfa..a489cd1ac8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -85,8 +85,6 @@ typedef struct SAggOperatorInfo { SExprSupp scalarExprSup; } SAggOperatorInfo; -int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } - static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void releaseQueryBuf(size_t numOfTables); @@ -106,7 +104,7 @@ void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); - pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start * 1000) / 1000.0; + pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } @@ -120,19 +118,21 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, b pOperator->pTaskInfo = pTaskInfo; } -int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { +int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, + __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, .cleanupFn = cleanup, .closeFn = closeFn, + .reqBufFn = reqBufFn, .getExplainFn = explain, }; @@ -939,10 +939,10 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u } static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, - const int32_t* rowCellOffset) { + const int32_t* rowEntryOffset) { bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { - struct SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowCellOffset); + SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (!isRowEntryInitialized(pResInfo)) { continue; } @@ -1145,43 +1145,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -// void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { -// return; -// } -// -// pQueryAttr->pos = 0; -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pTsdbReadHandle)) { -// if (isTaskKilled(pRuntimeEnv->qinfo)) { -// T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); -// } -// -// if (pQueryAttr->limit.offset > blockInfo.rows) { -// pQueryAttr->limit.offset -= blockInfo.rows; -// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; -// pTableQueryInfo->lastKey += step; -// -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, -// pQuery->limit.offset); -// } else { // find the appropriated start position in current block -// updateOffsetVal(pRuntimeEnv, &blockInfo); -// break; -// } -// } -// -// if (terrno != TSDB_CODE_SUCCESS) { -// T_LONG_JMP(pRuntimeEnv->env, terrno); -// } -// } - // static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, // STableQueryInfo* pTableQueryInfo) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1406,11 +1369,11 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; bool hasCountFunc = false; + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) { + const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; + if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || + (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { hasCountFunc = true; break; } @@ -1463,7 +1426,6 @@ static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppB *ppBlock = NULL; } - // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { if (OPTR_IS_OPENED(pOperator)) { @@ -1614,6 +1576,15 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) { taosMemoryFreeClear(pOperator); } +// each operator should be set their own function to return total cost buffer +int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { + if (pOperator->blocking) { + ASSERT(0); + } else { + return 0; + } +} + int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) { *defaultPgsz = 4096; while (*defaultPgsz < rowSize * 4) { @@ -1794,7 +1765,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -1871,8 +1842,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); -SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, - SExecTaskInfo* pTaskInfo); int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5ed26f627a..8d5af64777 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -381,7 +381,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1478,7 +1478,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pInfo->srcRowIndex = 0; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 4601175561..5d34064b7e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -470,7 +470,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -850,7 +850,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1142,7 +1142,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index e7cce39dfd..d460af971c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -136,7 +136,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index da77facb21..65bb40b195 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -118,8 +118,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, - destroyProjectOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL, + destroyProjectOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -415,7 +415,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1b05e8b573..0d245f0815 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -899,8 +899,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, - getTableScannerExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, + optrDefaultBufFn, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 pOperator->cost.openCost = 0; @@ -925,7 +925,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImpl, NULL, NULL, optrDefaultBufFn, NULL); return pOperator; } @@ -2139,7 +2139,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _end: @@ -2328,7 +2328,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -2465,7 +2465,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -2866,8 +2866,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, - getTableMergeScanExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, + optrDefaultBufFn, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; @@ -3015,7 +3015,7 @@ SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableC setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, NULL); + createOperatorFpSet(optrDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, optrDefaultBufFn, NULL); return pOperator; _error: diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index b355b85861..005b794f0b 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -75,7 +75,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, optrDefaultBufFn, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -521,8 +521,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, - getGroupSortExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, + optrDefaultBufFn, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -798,7 +798,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, - destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); + destroyMultiwayMergeOperatorInfo, optrDefaultBufFn, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 96f21babe6..32905ca897 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1437,7 +1437,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL); return pOperator; _error: @@ -1943,7 +1943,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _error: diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 90f5dde7c3..2374d80bbf 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -544,7 +544,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b0466b6216..2d0893b97b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1049,14 +1049,14 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SIntervalAggOperatorInfo* pInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; int32_t scanFlag = MAIN_SCAN; - - int64_t st = taosGetTimestampUs(); - SOperatorInfo* downstream = pOperator->pDownstream[0]; + int64_t st = taosGetTimestampUs(); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -1268,9 +1268,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { } SSDataBlock* pBlock = pInfo->binfo.pRes; - - ASSERT(pInfo->execModel == OPTR_EXEC_MODEL_BATCH); - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; @@ -1765,7 +1762,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->interval = interval; - pInfo->execModel = pTaskInfo->execModel; pInfo->twAggSup = as; pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; @@ -1797,7 +1793,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2015,7 +2011,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2088,7 +2084,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2749,7 +2745,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -3561,7 +3557,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); @@ -3705,8 +3701,8 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, + destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); } setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4066,7 +4062,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL); initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4314,7 +4310,6 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->win = pTaskInfo->window; iaInfo->inputOrder = TSDB_ORDER_ASC; iaInfo->interval = interval; - iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId; iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; @@ -4344,7 +4339,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4620,7 +4615,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pIntervalInfo->win = pTaskInfo->window; pIntervalInfo->inputOrder = TSDB_ORDER_ASC; pIntervalInfo->interval = interval; - pIntervalInfo->execModel = pTaskInfo->execModel; pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock; pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -4650,7 +4644,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4866,8 +4860,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, + destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index f06bafafe3..1c74d22a82 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -270,7 +270,7 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p #if __AVX2__ // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(int64_t); + int32_t width = (bitWidth >> 3u) / sizeof(int64_t); int32_t remainder = numOfRows % width; int32_t rounds = numOfRows / width; @@ -286,20 +286,11 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p } // let sum up the final results - if (type == TSDB_DATA_TYPE_BIGINT) { - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; + const int64_t* q = (const int64_t*)∑ + pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + rounds * width]; - } - } else { - const uint64_t* q = (const uint64_t*)∑ - pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.usum += (uint64_t)plist[j + rounds * width]; - } + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.isum += plist[j + rounds * width]; } #endif @@ -588,14 +579,14 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { const int64_t* plist = (const int64_t*) pCol->pData; // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { + if (simdAvailable && type == TSDB_DATA_TYPE_BIGINT) { i64VectorSumAVX2(plist, numOfRows, pAvgRes); } else { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (type == TSDB_DATA_TYPE_BIGINT) { pAvgRes->sum.isum += plist[i]; } else { - pAvgRes->sum.isum += (uint64_t)plist[i]; + pAvgRes->sum.usum += (uint64_t)plist[i]; } } } From 9e803332fce3d007fee29244b116eec10f8697bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 23:10:12 +0800 Subject: [PATCH 091/120] refact: adjust error codes --- include/util/taoserror.h | 94 +++++++++++++-------------- source/dnode/mnode/impl/src/mndAcct.c | 6 +- source/util/src/terror.c | 3 - 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2baf4d14e2..490dbf7744 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -40,61 +40,61 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc -// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x -// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x -// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x -// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //2.x -// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x -// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x -// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x -// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //2.x -// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //2.x -// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //2.x +// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) // 2.x +// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) // 2.x +// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) // 2.x +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) // 2.x +// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) // 2.x +// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) // 2.x +// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) // 2.x +// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) // 2.x +// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) // 2.x +// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) // 2.x #define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) -// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //2.x -// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //2.x -// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //2.x -// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //2.x -// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //2.x -// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x -// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x -#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) -// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //2.x +// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) // 2.x +// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) // 2.x +// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) // 2.x +// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) // 2.x +// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) // 2.x +// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) // 2.x +// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) // 2.x +#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) // +// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) // 2.x #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) -// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x -#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) -#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) -#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) +// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) // 2.x +#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) // +#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) // +#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) // //common & util -#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) -#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) +#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) // +// #define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) // 2.x #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102) // #define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x -#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) -#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) -#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) -#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) -#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) -#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) -#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) +#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) // +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) // internal +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) // internal +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) // internal +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) // internal +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) // internal +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) // internal -#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) -#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) -#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) -#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) -#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) -#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) +#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) // +#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) // internal +#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) // +// #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) +// #define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) +#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) // #define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) // -#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) -#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) -#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) -#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) -#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) -#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) -#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) -#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) -#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) +#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) // internal +#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) // +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) // internal +#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) // internal +#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) // internal +#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) // internal +#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) // internal +#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) // internal +#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) // internal #define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120) #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) // diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index ce2ec7b23a..148e21b507 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -220,7 +220,7 @@ static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } @@ -230,7 +230,7 @@ static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } @@ -240,7 +240,7 @@ static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2110c0b3aa..18eeca030e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -55,7 +55,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, "Operation not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "Out of Memory") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") @@ -68,8 +67,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") TAOS_DEFINE_ERROR(TSDB_CODE_APP_ERROR, "Unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE, "Out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Shared memory") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SHM_ID, "Invalid SHM ID") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG_LEN, "Invalid message len") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") From 3d28255dfae2ea98a1aa720502eafd5f2794a3b8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Dec 2022 23:19:57 +0800 Subject: [PATCH 092/120] change rpc log level --- source/libs/transport/src/transCli.c | 3 ++- source/libs/transport/src/transSvr.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a9afbd7ba8..4df2790959 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1519,7 +1519,8 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || + code == TSDB_CODE_VND_STOPPED) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 71e2e30511..73c6dc4011 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -250,9 +250,9 @@ static bool uvHandleReq(SSvrConn* pConn) { transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transMsg.code, (int)(cost)); } else { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", - transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, - transMsg.code, (int)(cost)); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, + transMsg.code, (int)(cost)); } } From 213b97b79ac2722db50cc029f363a88a84f0ac7d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 23:20:19 +0800 Subject: [PATCH 093/120] feat(api): taos_affected_rows64 add and modfiy tmsg.h --- include/libs/executor/dataSinkMgt.h | 2 +- source/client/src/clientMsgHandler.c | 2 +- source/dnode/vnode/src/tq/tqExec.c | 2 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 2 +- source/libs/qworker/src/qwMsg.c | 2 +- source/libs/qworker/src/qworker.c | 6 +++--- source/libs/scheduler/src/schJob.c | 2 +- source/libs/scheduler/src/schRemote.c | 2 +- source/libs/stream/src/streamDispatch.c | 4 ++-- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 8a02f372d1..ed7cbc8125 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -68,7 +68,7 @@ typedef struct SInputData { typedef struct SOutputData { int32_t numOfBlocks; - int32_t numOfRows; + int64_t numOfRows; // int32_t changed to int64_t int32_t numOfCols; int8_t compressed; char* pData; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4bd74a842f..d5a567075c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -450,7 +450,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; - (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 2514190035..093186ebbb 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -25,7 +25,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->precision = precision; pRetrieve->compressed = 0; pRetrieve->completed = 1; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d58c4dc6d3..7661944379 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -36,7 +36,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; - (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 410e62a18b..dd48023679 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1615,7 +1615,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } rsp->completed = 1; - rsp->numOfRows = htonl(rowNum); + rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index d9a7cea411..7154e22300 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -37,7 +37,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->precision = input->precision; rsp->compressed = input->compressed; rsp->compLen = htonl(len); - rsp->numOfRows = htonl(input->numOfRows); + rsp->numOfRows = htobe64(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); rsp->numOfBlocks = htonl(input->numOfBlocks); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 0890d10b65..fe2acb805b 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -275,7 +275,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, QW_ERR_RET(code); } - QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %d", pOutput->numOfBlocks, + QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %"PRId64, pOutput->numOfBlocks, pOutput->numOfRows); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); @@ -320,7 +320,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, pOutput->numOfBlocks++; if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %d", pOutput->numOfBlocks, + QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, pOutput->numOfRows); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); break; @@ -332,7 +332,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { - QW_TASK_DLOG("task fetched blocks %d rows %d reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); + QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); break; } } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index ea49b0c4e0..b9062859c8 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -526,7 +526,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); } int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { - SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + SCH_TASK_DLOG("got explain rsp, rows:%" PRId64 ", complete:%d", htobe64(pRsp->numOfRows), pRsp->completed); atomic_store_64(&pJob->resNumOfRows, htobe64(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 76ac0a2e9f..30622f96bb 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -114,7 +114,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); } - SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + SCH_TASK_DLOG("got fetch rsp, rows:%" PRId64 ", complete:%d", htobe64(rsp->numOfRows), rsp->completed); msg = NULL; schProcessOnDataFetched(pJob); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 2c36c299ee..4e0b0630bc 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -112,7 +112,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); pRetrieve->numOfCols = htonl(numOfCols); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); @@ -189,7 +189,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); From c20e6078259ba0090da7a0759f09acdb3d1a9f5e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 23:32:31 +0800 Subject: [PATCH 094/120] fix: compile error --- source/dnode/mnode/impl/test/acct/acct.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 46a9a465eb..1f59a7fcca 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } TEST_F(MndTestAcct, 02_Alter_Acct) { @@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } TEST_F(MndTestAcct, 03_Drop_Acct) { @@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } From de0022b473a2f19093ee7cbccbf3f5df36c35370 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 23:38:05 +0800 Subject: [PATCH 095/120] fix(query): fix error in record the created time. --- source/libs/executor/src/executor.c | 4 ++-- source/libs/executor/src/executorimpl.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 60080d204b..4ad81f609a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -207,7 +207,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n } setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->cost.created = taosGetTimestampMs(); + pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE; pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo); if (NULL == pTaskInfo->pRoot) { @@ -711,7 +711,7 @@ static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " + "%s :cost summary: idle:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " "createGroupIdMap:%.2f ms, total blocks:%d, " "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, GET_TASKID(pTaskInfo), idleTime / 1000.0, pSummary->elapsedTime / 1000.0, pSummary->extractListTime, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a489cd1ac8..b408e32061 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1826,7 +1826,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->schemaInfo.dbname = strdup(dbFName); - pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; pTaskInfo->execModel = model; pTaskInfo->pTableInfoList = tableListCreate(); @@ -2333,6 +2332,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } + (*pTaskInfo)->cost.created = taosGetTimestampUs(); return TSDB_CODE_SUCCESS; _complete: From 69eae060665f184729ad0a6d72af9d36377bf4e1 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Fri, 2 Dec 2022 23:45:55 +0800 Subject: [PATCH 096/120] enh(taosAdapter): support loongArch64 --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index cc46ef9938..2a7b76d768 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ff7de07 + GIT_TAG e07f41b SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From e97c8f637ba4506078e1018be1c241346f9b11da Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 3 Dec 2022 08:25:57 +0800 Subject: [PATCH 097/120] enh: logging sync log buffer rollback events --- source/libs/sync/inc/syncPipeline.h | 2 +- source/libs/sync/src/syncPipeline.c | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 4208d40a69..8c7edf85ff 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -107,7 +107,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode); // private SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex index, bool* pInBuf); int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); -int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex); +int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f0be976402..fcb0d63209 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -310,7 +310,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt ASSERT(pEntry->index == pExist->index); if (pEntry->term != pExist->term) { - (void)syncLogBufferRollback(pBuf, index); + (void)syncLogBufferRollback(pBuf, pNode, index); } else { sTrace("vgId:%d, duplicate log entry received. index: %" PRId64 ", term: %" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -971,14 +971,18 @@ void syncLogBufferDestroy(SSyncLogBuffer* pBuf) { return; } -int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex) { +int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex) { ASSERT(pBuf->commitIndex < toIndex && toIndex <= pBuf->endIndex); + sInfo("vgId:%d, rollback sync log buffer. toindex: %" PRId64 ", buffer: [%" PRId64 " %" PRId64 " %" PRId64 + ", %" PRId64 ")", + pNode->vgId, toIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + SyncIndex index = pBuf->endIndex - 1; while (index >= toIndex) { SSyncRaftEntry* pEntry = pBuf->entries[index % pBuf->size].pItem; if (pEntry != NULL) { - syncEntryDestroy(pEntry); + (void)syncEntryDestroy(pEntry); pEntry = NULL; memset(&pBuf->entries[index % pBuf->size], 0, sizeof(pBuf->entries[0])); } @@ -996,7 +1000,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { ASSERT(lastVer == pBuf->matchIndex); SyncIndex index = pBuf->endIndex - 1; - (void)syncLogBufferRollback(pBuf, pBuf->matchIndex + 1); + (void)syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1); sInfo("vgId:%d, reset sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); From 2ce07c6ec1720c28dd6278020b86f70d716447b0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 3 Dec 2022 08:51:58 +0800 Subject: [PATCH 098/120] fix: get result of tmsPutToQueue for FpCommitCb --- include/libs/sync/sync.h | 4 ++-- source/dnode/mnode/impl/src/mndSync.c | 9 ++++++--- source/dnode/vnode/src/vnd/vnodeSync.c | 13 +++++++------ source/libs/sync/src/syncPipeline.c | 9 +++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 4741f7f3d8..1fa862b145 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -138,8 +138,8 @@ typedef struct SSnapshotMeta { typedef struct SSyncFSM { void* data; - void (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); - void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 10cd6416b4..2e3c04eb78 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -72,7 +72,7 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } -void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SMnode *pMnode = pFsm->data; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; @@ -114,12 +114,15 @@ void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *p mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } } + + return 0; } -void mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - mndProcessWriteMsg(pFsm, pMsg, pMeta); +int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { + int32_t code = mndProcessWriteMsg(pFsm, pMsg, pMeta); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; + return code; } int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index bd8b00edf0..55ba0d3e6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -295,7 +295,7 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot) return 0; } -static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; pMsg->info.conn.applyIndex = pMeta->index; pMsg->info.conn.applyTerm = pMeta->term; @@ -306,17 +306,18 @@ static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbM pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); + return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } -static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); +static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } -static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { if (pMeta->isWeak == 1) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } + return 0; } static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index fcb0d63209..d5d77274a9 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -457,9 +457,9 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn cbMeta.flag = -1; (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); ASSERT(rpcMsg.pCont == NULL); - return 0; + return code; } int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf) { @@ -516,8 +516,9 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { - sError("vgId:%d, failed to execute sync log entry in FSM. log index:%" PRId64 ", term:%" PRId64 "", vgId, - pEntry->index, pEntry->term); + sError("vgId:%d, failed to commit sync log entry. index:%" PRId64 ", term:%" PRId64 + ", role: %d, current term: %" PRId64, + vgId, pEntry->index, pEntry->term, role, term); goto _out; } pBuf->commitIndex = index; From 81dc25819d6480199c915f13587c5f70415b1e4a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 3 Dec 2022 09:42:58 +0800 Subject: [PATCH 099/120] feat(api): add int64 affect row fix build error --- source/libs/qworker/src/qworker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index fe2acb805b..e4c3c95b9c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -327,7 +327,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } if (0 == ctx->level) { - QW_TASK_DLOG("task fetched blocks %d rows %d, level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); + QW_TASK_DLOG("task fetched blocks %d rows %"PRId64", level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); break; } From 6cb65fabdc907105f7477325f0d516417236c5bb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 3 Dec 2022 09:43:37 +0800 Subject: [PATCH 100/120] fix: fix stable not exist crash issue --- source/libs/catalog/src/ctgCache.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 4626d4527d..6d28420f79 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -439,6 +439,12 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** ctx->tbInfo.tbType, dbFName); ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + taosMemoryFreeClear(*pTableMeta); + ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); + return TSDB_CODE_SUCCESS; + } STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { From d9c261cd13a5d34300b195ae6afccdbd1a96d540 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 3 Dec 2022 10:03:18 +0800 Subject: [PATCH 101/120] refact: remove some error codes --- include/util/taoserror.h | 56 +++++++++++---- source/client/src/clientEnv.c | 6 +- source/client/src/clientHb.c | 20 +++--- source/client/src/clientImpl.c | 18 ++--- source/client/src/clientMsgHandler.c | 2 +- source/client/src/clientRawBlockWrite.c | 16 ++--- source/client/src/clientSml.c | 10 +-- source/client/src/clientStmt.c | 28 ++++---- source/common/src/tdataformat.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- source/dnode/mnode/sdb/src/sdbFile.c | 2 +- source/dnode/mnode/sdb/src/sdbHash.c | 4 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgRemote.c | 4 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 50 ++++++------- source/libs/executor/src/dataDeleter.c | 4 +- source/libs/executor/src/dataDispatcher.c | 12 ++-- source/libs/executor/src/dataInserter.c | 10 +-- source/libs/executor/src/exchangeoperator.c | 8 +-- source/libs/executor/src/executil.c | 4 +- source/libs/executor/src/executor.c | 20 +++--- source/libs/executor/src/executorimpl.c | 18 ++--- source/libs/executor/src/groupoperator.c | 4 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/sysscanoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 42 +++++------ source/libs/executor/src/tsort.c | 2 +- source/libs/function/src/udfd.c | 2 +- source/libs/index/src/indexFilter.c | 28 ++++---- source/libs/index/src/indexTfile.c | 2 +- source/libs/nodes/src/nodesToSQLFuncs.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parInsertSml.c | 6 +- source/libs/parser/src/parInsertSql.c | 6 +- source/libs/parser/src/parInsertStmt.c | 4 +- source/libs/parser/src/parInsertUtil.c | 18 ++--- source/libs/parser/src/parTranslater.c | 2 +- source/libs/parser/src/parUtil.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qcom/src/queryUtil.c | 8 +-- source/libs/qcom/src/querymsg.c | 8 +-- source/libs/qworker/src/qwDbg.c | 18 ++--- source/libs/qworker/src/qwMsg.c | 28 ++++---- source/libs/qworker/src/qwUtil.c | 14 ++-- source/libs/qworker/src/qworker.c | 22 +++--- source/libs/scalar/src/filter.c | 30 ++++---- source/libs/scalar/src/scalar.c | 46 ++++++------ source/libs/scalar/src/sclvector.c | 8 +-- source/libs/scheduler/src/schFlowCtrl.c | 8 +-- source/libs/scheduler/src/schJob.c | 52 +++++++------- source/libs/scheduler/src/schRemote.c | 70 +++++++++---------- source/libs/scheduler/src/schTask.c | 24 +++---- source/libs/scheduler/src/schUtil.c | 2 +- source/libs/scheduler/src/scheduler.c | 6 +- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/wal/src/walMeta.c | 8 +-- source/libs/wal/src/walMgmt.c | 2 +- source/libs/wal/src/walRead.c | 8 +-- source/util/src/terror.c | 12 ---- tests/parallel_test/cases.task | 1 - 65 files changed, 421 insertions(+), 402 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 490dbf7744..e8e06f4af1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -126,11 +126,11 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) #define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) #define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) -#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) +// #define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) // 2.x #define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) #define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) #define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) -#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) +// #define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) // 2.x #define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) #define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) #define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) @@ -158,9 +158,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) // mnode-common +// #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x +// #define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) // 2.x +// #define TSDB_CODE_MND_ACTION_NEED_REPROCESSEDTAOS_DEF_ERROR_CODE(0, 0x0302) // 2.x #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) +// #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) // 2.x #define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) +// #define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) // 2.x +// #define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) // 2.x +// #define TSDB_CODE_MND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x030A) // 2.x #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) #define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) #define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) @@ -174,7 +182,7 @@ int32_t* taosGetErrno(); // mnode-sdb #define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // internal +// #define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // 2.x #define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) #define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) #define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) @@ -217,22 +225,31 @@ int32_t* taosGetErrno(); // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +// #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) // 2.x #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0362) +// #define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) // 2.x #define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x0365) +// #define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) // 2.x +// #define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) // 2.x +// #define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) // 2.x #define TSDB_CODE_MND_TAG_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) #define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) +// #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) // mnode-func #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) +// #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) // 2.x #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) #define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0374) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) +// #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) // 2.x +// #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) // 2.x #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0378) #define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x0379) @@ -390,8 +407,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) #define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) #define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) +// #define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) // 2.x +// #define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) // 2.x #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) @@ -405,6 +422,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) +// #define TSDB_CODE_TDB_INCOMPLETE_DFILESET TAOS_DEF_ERROR_CODE(0, 0x0617) // 2.x #define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0618) #define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0619) #define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A) @@ -415,8 +433,9 @@ int32_t* taosGetErrno(); // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) -#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) -#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) +// #define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) // 2.x +// #define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) // 2.x +// #define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) // 2.x #define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) #define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) #define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) @@ -428,6 +447,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) #define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070E) #define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) +// #define TSDB_CODE_QRY_INVALID_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0710) // 2.x +// #define TSDB_CODE_QRY_RESULT_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0711) // 2.x #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0720) #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0721) #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0722) @@ -463,7 +484,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D) // sync +// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x +// #define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) // 2.x +// #define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) // 2.x #define TSDB_CODE_SYN_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0903) +// #define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) // 2.x +// #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x +// #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x +// #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x #define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) #define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x090C) #define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) @@ -481,7 +512,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) #define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) #define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) -#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) +// #define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) #define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) #define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) #define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) @@ -492,16 +523,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_NO_COMMITTED_OFFSET TAOS_DEF_ERROR_CODE(0, 0x0A0D) // wal -#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) +// #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) // 2.x #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) -#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) +// #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // 2.x #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) #define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) #define TSDB_CODE_WAL_LOG_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x1007) // tfs +// #define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) // 2.x #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) #define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) #define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) @@ -510,7 +542,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) -#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x220F) +// #define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x220F) // 2.x // catalog #define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6f1414b72d..f4b18a373b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -244,14 +244,14 @@ void destroyTscObj(void *pObj) { void *createTscObj(const char *user, const char *auth, const char *db, int32_t connType, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pObj->pRequests = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (NULL == pObj->pRequests) { taosMemoryFree(pObj); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -281,7 +281,7 @@ int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, ri void *createRequest(uint64_t connId, int32_t type, int64_t reqid) { SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6bdc835217..47ed2cf035 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -69,7 +69,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } else { SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == vgInfo) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -82,7 +82,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo->vgHash) { taosMemoryFree(vgInfo); tscError("hash init[%d] failed", rsp->vgNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -92,7 +92,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo->vgHash); taosMemoryFree(vgInfo); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } } @@ -366,7 +366,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc)); if (NULL == desc.subDesc) { releaseRequest(*rid); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } code = schedulerGetTasksStatus(pRequest->body.queryJob, desc.subDesc); @@ -394,14 +394,14 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); if (NULL == pTscObj) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SQueryHbReqBasic *hbBasic = (SQueryHbReqBasic *)taosMemoryCalloc(1, sizeof(SQueryHbReqBasic)); if (NULL == hbBasic) { tscError("calloc %d failed", (int32_t)sizeof(SQueryHbReqBasic)); releaseTscObj(connKey->tscRid); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } hbBasic->connId = pTscObj->connId; @@ -419,7 +419,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { tscWarn("taosArrayInit %d queryDesc failed", numOfQueries); releaseTscObj(connKey->tscRid); taosMemoryFree(hbBasic); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); @@ -613,7 +613,7 @@ static FORCE_INLINE void hbMgrInitHandle() { SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); @@ -737,7 +737,7 @@ static void *hbThreadFunc(void *param) { int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosArrayPush(mgr, &pAppHbMgr); @@ -748,7 +748,7 @@ static void *hbThreadFunc(void *param) { SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 846accf51b..920bac0e2b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -166,7 +166,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); @@ -179,7 +179,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (pParam == NULL) { destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } tsem_init(&pParam->sem, 0, 0); @@ -198,7 +198,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, taosMemoryFree(param); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } (*pRequest)->allocatorRefId = -1; @@ -209,7 +209,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -609,7 +609,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } default: tscError("unknown query policy: %d", tsQueryPolicy); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayDestroy(pDbVgList); @@ -670,7 +670,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } default: tscError("unknown query policy: %d", tsQueryPolicy); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } _return: @@ -1136,7 +1136,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { int32_t tblNum = taosArrayGetSize(pRequest->tableList); if (dbNum <= 0 && tblNum <= 0) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1231,7 +1231,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t SAppInstInfo* pAppInfo, int connType) { STscObj* pTscObj = createTscObj(user, auth, db, connType, pAppInfo); if (NULL == pTscObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return pTscObj; } @@ -1268,7 +1268,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4bd74a842f..7d0d9574f6 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -86,7 +86,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { /*assert(connectRsp.epSet.numOfEps > 0);*/ if (connectRsp.epSet.numOfEps == 0) { - setErrno(pRequest, TSDB_CODE_MND_APP_ERROR); + setErrno(pRequest, TSDB_CODE_APP_ERROR); tsem_post(&pRequest->body.rspSem); goto End; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 5583c8ae00..150194aa27 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1388,7 +1388,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vgData; @@ -1579,7 +1579,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vgData; @@ -1726,7 +1726,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; void* tmp = taosMemoryRealloc(vgData.data, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -1737,7 +1737,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t totalLen = sizeof(SSubmitReq) + submitLen; void* tmp = taosMemoryCalloc(1, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -1840,7 +1840,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { while (vData) { SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vData->vg; @@ -2029,7 +2029,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; void* tmp = taosMemoryRealloc(vgData.data, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -2040,7 +2040,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) int32_t totalLen = sizeof(SSubmitReq) + submitLen; void* tmp = taosMemoryCalloc(1, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -2146,7 +2146,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) while (vData) { SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vData->vg; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 719a6f82d2..a4e943da32 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1417,7 +1417,7 @@ static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *col SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (!kvHash) { uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); @@ -2092,7 +2092,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int l cols = taosArrayInit(16, POINTER_BYTES); if (cols == NULL) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data cols = info->colsContainer; @@ -2121,7 +2121,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int l if (!tinfo) { smlDestroyCols(cols); if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); oneTable = &tinfo; @@ -2192,13 +2192,13 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { int ret = TSDB_CODE_SUCCESS; SSmlTableInfo *tinfo = smlBuildTableInfo(); if (!tinfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SArray *cols = taosArrayInit(16, POINTER_BYTES); if (cols == NULL) { uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 291e0f6ae3..82ea9e0d8f 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -83,7 +83,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { } break; default: - code = TSDB_CODE_TSC_APP_ERROR; + code = TSDB_CODE_APP_ERROR; break; } @@ -118,7 +118,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); if (NULL == pRes->fields || NULL == pRes->userFields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -136,7 +136,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); } @@ -144,7 +144,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); } @@ -463,7 +463,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } pStmt->bInfo.needParse = false; @@ -512,7 +512,7 @@ int32_t stmtResetStmt(STscStmt* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -527,13 +527,13 @@ TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid) { pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); if (NULL == pStmt) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pStmt); return NULL; } @@ -618,7 +618,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } tscDebug("start to bind stmt tag values"); @@ -641,7 +641,7 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); @@ -659,7 +659,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields)); @@ -733,7 +733,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } if (colIdx < 0) { @@ -745,7 +745,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } else { if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { tscError("bind column index not in sequence"); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } pStmt->bInfo.sBindLastIdx = colIdx; @@ -897,7 +897,7 @@ _return: if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { if (NULL == pRsp) { tscError("no submit resp got for auto create table"); - code = TSDB_CODE_TSC_APP_ERROR; + code = TSDB_CODE_APP_ERROR; } else { code = stmtUpdateTableUid(pStmt, pRsp); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 728f669fc8..fc49c03379 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -518,7 +518,7 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); if (pIter == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d42b66455f..fe76ba31d7 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -542,7 +542,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } default: mError("invalid kv key:%d", kv->key); - hbRsp.status = TSDB_CODE_MND_APP_ERROR; + hbRsp.status = TSDB_CODE_APP_ERROR; break; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 161bf7536f..b4ea8ee125 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -276,7 +276,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg } if (createReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -376,7 +376,7 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p } if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6558a98aaa..318a746bb0 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -415,7 +415,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { break; } } else { - code = TSDB_CODE_SDB_APP_ERROR; + code = TSDB_CODE_APP_ERROR; taosHashCancelIterate(hash, ppRow); break; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c44b659ef5..32b34ea3a3 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -108,7 +108,7 @@ static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) { SHashObj *hash = pSdb->hashObjs[type]; if (hash == NULL) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -302,7 +302,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { terrno = TSDB_CODE_SDB_OBJ_DROPPING; break; default: - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; break; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 003b387838..191fc1b49c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -75,7 +75,7 @@ static void tqPushEntryFree(void* data) { STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d9f0c55d24..8211e58279 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1311,7 +1311,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); if (buf == NULL) { cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_TDB_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; @@ -1354,7 +1354,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_TDB_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t numOfTotal = 0; @@ -3812,7 +3812,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL tsdbReaderClose(p); *ppReader = NULL; - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e0c0e90d1..55703002b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -107,7 +107,7 @@ int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItem SArray *pArray = taosArrayInit(pMapData->nItem, itemSize); if (pArray == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d1551..1d74957718 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -619,7 +619,7 @@ int32_t catalogInit(SCatalogCfg* cfg) { gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); - CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 753c4ce917..7cc6c90d30 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -384,13 +384,13 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArr SMsgSendInfo* msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); if (NULL == param) { qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); - CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } param->reqType = msgType; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d58c4dc6d3..6713d0cbe9 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -497,7 +497,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN); if (NULL == buf2) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 03c7249294..f9b166de04 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -77,19 +77,19 @@ int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, d SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { qError("calloc SExplainCtx failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); if (NULL == tbuf) { qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } ctx->mode = mode; @@ -235,14 +235,14 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo } default: qError("not supported physical node type %d", pNode->type); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } if (pPhysiChildren) { *pChildren = nodesMakeList(); if (NULL == *pChildren) { qError("nodesMakeList failed"); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -260,7 +260,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); if (NULL == (*pExecInfo)) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SExplainRsp *rsp = NULL; @@ -272,7 +272,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); @@ -281,7 +281,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S rsp = taosArrayGet(group->nodeExecInfo, i); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); @@ -297,13 +297,13 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai if (NULL == pNode) { *pResNode = NULL; qError("physical node is NULL"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = 0; @@ -378,7 +378,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { qError("taosMemoryMalloc %d failed", len); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(row.buf, tbuf, len); @@ -389,7 +389,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -407,7 +407,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SPhysiNode *pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } switch (pNode->type) { @@ -793,7 +793,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcStartGroupId, sizeof(pExchNode->srcStartGroupId)); if (NULL == group) { qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } nodeNum += group->nodeNum; @@ -1585,7 +1585,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } default: qError("not supported physical node type %d", pNode->type); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1594,7 +1594,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t code = 0; @@ -1614,7 +1614,7 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, b SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } group->singleChannel = singleChannel; @@ -1636,7 +1636,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } SSDataBlock *pBlock = createDataBlock(); @@ -1659,7 +1659,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); blockDataDestroy(pBlock); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } rsp->completed = 1; @@ -1698,7 +1698,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == groupHash) { qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } QRY_ERR_JRET( @@ -1732,7 +1732,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -1800,7 +1800,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t if (NULL == group) { qError("group %d not in groupHash", groupId); tFreeSExplainRsp(pRspMsg); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } taosWLockLatch(&group->lock); @@ -1811,7 +1811,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } group->physiPlanExecNum = pRspMsg->numOfPlans; @@ -1821,7 +1821,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } if (group->physiPlanExecNum != pRspMsg->numOfPlans) { @@ -1830,7 +1830,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index c7a2480204..12fb449238 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -135,12 +135,12 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterBuf* pBuf = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM); if (NULL == pBuf) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (!allocBuf(pDeleter, pInput, pBuf)) { taosFreeQitem(pBuf); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } toDataCacheEntry(pDeleter, pInput, pBuf); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d4248fc420..c45226e02b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -128,12 +128,12 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); if (NULL == pBuf) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (!allocBuf(pDispatcher, pInput, pBuf)) { taosFreeQitem(pBuf); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } toDataCacheEntry(pDispatcher, pInput, pBuf); @@ -237,8 +237,8 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } dispatcher->sink.fPut = putDataBlock; dispatcher->sink.fEndPut = endPut; @@ -254,8 +254,8 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { taosMemoryFree(dispatcher); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *pHandle = dispatcher; return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 09ca1d27b9..346fcc9729 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -101,7 +101,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { taosMemoryFreeClear(pMsg); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -304,8 +304,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat void* pParam) { SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SQueryInserterNode* pInserterNode = (SQueryInserterNode*)pDataSink; @@ -342,8 +342,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat if (NULL == inserter->pDataBlocks) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 76e27f12fc..3b5928a284 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -424,20 +424,20 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } void* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); taosMemoryFree(msg); return pTaskInfo->code; @@ -453,7 +453,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas taosMemoryFreeClear(msg); taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return pTaskInfo->code; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1241bdf5a4..93c4372586 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1457,7 +1457,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu SqlFunctionCtx* p = NULL; SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); if (pValCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1605,7 +1605,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols); if (pCond->colList == NULL || pCond->pSlotList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pCond->colList); taosMemoryFreeClear(pCond->pSlotList); return terrno; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4ad81f609a..6fca2858dc 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,12 +35,12 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetSMABlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); @@ -76,12 +76,12 @@ static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetStreamOpOpen(pOperator->pDownstream[0], id); @@ -95,12 +95,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); @@ -139,7 +139,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; @@ -156,7 +156,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pBlocks == NULL || numOfBlocks == 0) { @@ -177,7 +177,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pBlocks == NULL || numOfBlocks == 0) { @@ -980,7 +980,7 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s if (pCond->colList == NULL || pCond->pSlotList == NULL) { taosMemoryFreeClear(pCond->colList); taosMemoryFreeClear(pCond->pSlotList); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b408e32061..bd22e864cd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -675,7 +675,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); } } } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate @@ -726,7 +726,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); } } } @@ -2196,12 +2196,12 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { qError("join not supported for stream block scan"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); } else { @@ -2219,13 +2219,13 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return 0; } else { ASSERT(0); - terrno = TSDB_CODE_QRY_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return -1; } } else { if (LIST_LENGTH(pNode->pChildren) != 1) { ASSERT(0); - terrno = TSDB_CODE_QRY_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return -1; } SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pNode->pChildren, 0); @@ -2253,7 +2253,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa if (pTableScanInfo->dataReader == NULL) { ASSERT(0); qError("failed to create data reader"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } // TODO: set uid and ts to data reader return 0; @@ -2428,7 +2428,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pExecInfoList); if (code != TSDB_CODE_SUCCESS) { // taosMemoryFreeClear(*pRes); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -2445,7 +2445,7 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul int32_t size = pAggSup->resultRowSize; if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *pResult = (SResultRow*)value; ASSERT(*pResult); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 5d34064b7e..2cd1bd7dec 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -310,7 +310,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } int32_t rowIndex = j - num; @@ -327,7 +327,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } int32_t rowIndex = pBlock->info.rows - num; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0d245f0815..04f5f4ecfe 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2183,7 +2183,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 32905ca897..a88f673e0b 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1335,7 +1335,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -1881,7 +1881,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)); if (pCond->colList == NULL || pCond->pSlotList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2d0893b97b..d9a011a892 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -648,7 +648,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); @@ -927,7 +927,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = @@ -943,7 +943,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } // window start key interpolation @@ -967,7 +967,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -1150,7 +1150,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); @@ -1175,7 +1175,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); @@ -1855,7 +1855,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } // pInfo->numOfRows data belong to the current session window @@ -1874,7 +1874,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); @@ -2164,7 +2164,7 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup); if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } num++; @@ -2403,7 +2403,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } if (IS_FINAL_OP(pInfo)) { @@ -2574,7 +2574,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { for (int32_t i = 0; i < chIndex + 1 - size; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; @@ -3000,7 +3000,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); @@ -3109,13 +3109,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData pAggSup->pResultRows, pStUpdated, pStDeleted); // coverity scan error if (!winInfo.pOutputBuf) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); saveSessionOutputBuf(pAggSup, &winInfo); @@ -3123,7 +3123,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -3447,7 +3447,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } taosArrayPush(pInfo->pChildren, &pChildOp); } @@ -3890,14 +3890,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } saveSessionOutputBuf(pAggSup, &curWin.winInfo); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curWin.winInfo, pSeUpdated); if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } @@ -4448,7 +4448,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; @@ -4465,7 +4465,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } // window start key interpolation @@ -4494,7 +4494,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 37688c0b53..30911887bb 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -156,7 +156,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); if (pSource == NULL) { taosArrayDestroy(pPageIdList); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pSource->src.pBlock = pBlock; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 9d249e0831..afadf7401d 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -419,7 +419,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (connectRsp.epSet.numOfEps == 0) { - msgInfo->code = TSDB_CODE_MND_APP_ERROR; + msgInfo->code = TSDB_CODE_APP_ERROR; goto _return; } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 72828e1daa..5a86bc8678 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -195,7 +195,7 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { } char *tv = taosMemoryCalloc(1, valLen + 1); if (tv == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(tv, pData, valLen); @@ -259,7 +259,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -269,7 +269,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIFParam *res = (SIFParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { indexError("no result for node, type:%d, node:%p", nodeType(node), node); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } *param = *res; break; @@ -300,7 +300,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && @@ -319,7 +319,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); // if (paramList[0].colValType == TSDB_DATA_TYPE_JSON && // ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) { - // return TSDB_CODE_QRY_OUT_OF_MEMORY; + // return TSDB_CODE_OUT_OF_MEMORY; //} } *params = paramList; @@ -335,7 +335,7 @@ static int32_t sifInitParamList(SIFParam **params, SNodeList *nodeList, SIFCtx * SIFParam *tParams = taosMemoryCalloc(nodeList->length, sizeof(SIFParam)); if (tParams == NULL) { indexError("failed to calloc, nodeList: %p", nodeList); - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SListCell *cell = nodeList->pHead; @@ -464,7 +464,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); if (tm == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); @@ -722,7 +722,7 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; @@ -740,7 +740,7 @@ static EDealRes sifWalkLogic(SNode *pNode, void *context) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; @@ -756,7 +756,7 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) { return DEAL_RES_ERROR; } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -807,7 +807,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); @@ -821,7 +821,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } if (res->result != NULL) { taosArrayAddAll(pDst->result, res->result); @@ -844,7 +844,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); @@ -856,7 +856,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } *status = res->status; sifFreeParam(res); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e002ff9c32..b34d05d297 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -269,7 +269,7 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { if (ret != 0) { taosArrayDestroy(offsets); indexError("failed to find target tablelist"); - return TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FILE_CORRUPTED; } } taosArrayDestroy(offsets); diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 9325d02886..0181da92a9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -132,7 +132,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { char *t = nodesGetStrValueFromNode(colNode); if (NULL == t) { nodesError("fail to get str value from valueNode"); - NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + NODES_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t tlen = strlen(t); @@ -229,5 +229,5 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); - NODES_RET(TSDB_CODE_QRY_APP_ERROR); + NODES_RET(TSDB_CODE_APP_ERROR); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f363cd0f10..de51e9d9f3 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1563,7 +1563,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { pNode->datum.p = (char*)value; break; default: - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index e76ca7751d..358baa74cb 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -114,7 +114,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS if (!isOrdered) { pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; for (col_id_t i = 0; i < pColList->numOfBound; ++i) { @@ -150,11 +150,11 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); if (!*tagName) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 064e376894..df7a3953f2 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -236,7 +236,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b if (!isOrdered) { pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; for (col_id_t i = 0; i < pColList->numOfBound; ++i) { @@ -1261,7 +1261,7 @@ static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, // do nothing, if allocate more memory failed pDataBlock->nAllocSize = nAllocSizeOld; *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -1524,7 +1524,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index e0e191b7c9..4ed72e6c14 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -50,7 +50,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch int32_t code = TSDB_CODE_SUCCESS; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); @@ -339,7 +339,7 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 0600accd6d..73cedfeb3d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -198,7 +198,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } dataBuf->nAllocSize = (uint32_t)defaultSize; @@ -212,7 +212,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { taosMemoryFreeClear(dataBuf); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); @@ -249,7 +249,7 @@ int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbR memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); } else { pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -350,7 +350,7 @@ static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; @@ -518,7 +518,7 @@ static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* p if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; @@ -668,7 +668,7 @@ int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { insDestroyBlockArrayList(pVnodeDataBlockList); taosMemoryFreeClear(dataBuf->pData); taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -721,7 +721,7 @@ int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { } else { // do nothing, if allocate more memory failed pDataBlock->nAllocSize = nAllocSizeOld; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -938,13 +938,13 @@ int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray* size_t numOfVg = taosArrayGetSize(pVgDataBlocks); *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); if (NULL == *pDataBlocks) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bca139664d..049d18e20a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5682,7 +5682,7 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { if (s != *len) { taosCloseFile(&tfile); taosMemoryFreeClear(*buf); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosCloseFile(&tfile); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d9fc83c20f..fa091901b6 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -377,7 +377,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi int32_t valLen = (int32_t)strlen(jsonValue); char* tmp = taosMemoryCalloc(1, valLen * TSDB_NCHAR_SIZE); if (!tmp) { - retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + retCode = TSDB_CODE_OUT_OF_MEMORY; goto end; } val.type = TSDB_DATA_TYPE_NCHAR; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index bed8f82d3a..9cc55a7cd5 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -432,7 +432,7 @@ class MockCatalogServiceImpl { int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); if (!dst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(dst->get(), src, len); return TSDB_CODE_SUCCESS; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 6eadf80e3d..938be3fcbd 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -159,7 +159,7 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); destroySendMsgInfo(pInfo); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -441,7 +441,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { int32_t metaSize = sizeof(STableMeta) + numOfField * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, metaSize); return TSDB_CODE_SUCCESS; @@ -455,7 +455,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, sizeof(*pSrc)); if (pSrc->vgHash) { @@ -463,7 +463,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { HASH_ENTRY_LOCK); if (NULL == (*pDst)->vgHash) { taosMemoryFreeClear(*pDst); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SVgroupInfo* vgInfo = NULL; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index fadc39f21d..2e4b000761 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -34,7 +34,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; @@ -52,7 +52,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->vgHash = taosHashInit(usedbRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < usedbRsp->vgNum; ++i) { @@ -61,7 +61,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { qDebug("the %dth vgroup, id %d, epNum %d, current %s port %d", i, pVgInfo->vgId, pVgInfo->epSet.numOfEps, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].fqdn, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].port); if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -387,7 +387,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pTableMeta->vgId = isStb ? 0 : msg->vgId; diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index a122508f98..db6e5b19fb 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -28,57 +28,57 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, return TSDB_CODE_SUCCESS; } - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_INIT) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_INIT: if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_EXEC: if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_SUCC: if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_FAIL) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_FAIL: if (newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_DROP: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_PART_SUCC) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; default: QW_TASK_ELOG("invalid task origStatus:%s", jobTaskStatusStr(oriStatus)); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 7dc13ce5f3..d77dc2b3f3 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -17,7 +17,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r (SRetrieveTableRsp *)(rpcMalloc ? rpcReallocCont(*rsp, msgSize) : taosMemoryRealloc(*rsp, msgSize)); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } if (NULL == *rsp) { @@ -78,18 +78,18 @@ int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp); if (msgSize < 0) { qError("tSerializeSQueryTableRsp failed"); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } void *pRsp = rpcMallocCont(msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) { qError("tSerializeSQueryTableRsp %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg rpcRsp = { @@ -212,19 +212,19 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg pNewMsg = { @@ -250,7 +250,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { SQueryContinueReq *req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } req->header.vgId = mgmt->nodeId; @@ -291,19 +291,19 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg brokenMsg = { @@ -327,17 +327,17 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo * int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_ELOG("calloc %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg brokenMsg = { diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 86fd1d533c..ce3b493638 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -76,7 +76,7 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { QW_SCH_ELOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } QW_LOCK(QW_WRITE, &mgmt->schLock); @@ -87,7 +87,7 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } taosHashCleanup(newSch.tasksHash); @@ -114,7 +114,7 @@ int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQW QW_RET(TSDB_CODE_QRY_SCH_NOT_EXIST); } else { QW_SCH_ELOG("unknown notExistOpt:%d", nOpt); - QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_RET(TSDB_CODE_APP_ERROR); } } @@ -171,7 +171,7 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i } } else { QW_TASK_ELOG("taosHashPut to tasksHash failed, error:%x - %s", code, tstrerror(code)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } QW_UNLOCK(QW_WRITE, &sch->tasksLock); @@ -251,7 +251,7 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { } } else { QW_TASK_ELOG("taosHashPut to ctxHash failed, error:%x", code); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -363,7 +363,7 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove task from hash failed"); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } QW_TASK_DLOG_E("task status dropped"); @@ -509,7 +509,7 @@ int32_t qwOpenRef(void) { if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } } taosWUnLockLatch(&gQwMgmt.lock); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9a318df324..f4fb3d91fd 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -216,7 +216,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) if (NULL == hbInfo->rsp.taskStatus) { QW_UNLOCK(QW_READ, &sch->tasksLock); QW_ELOG("taosArrayInit taskStatus failed, num:%d", taskNum); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } void *key = NULL; @@ -463,7 +463,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } default: QW_TASK_ELOG("invalid phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } if (ctx->rspCode) { @@ -508,7 +508,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_PHASE_POST_FETCH == phase) { QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } // qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); @@ -629,7 +629,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); @@ -1059,7 +1059,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } ctx.taskHandle = pTaskInfo; @@ -1097,7 +1097,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorker)); atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; @@ -1109,20 +1109,20 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (NULL == mgmt->schHash) { taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == mgmt->ctxHash) { qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->timer = taosTmrInit(0, 0, 0, "qworker"); if (NULL == mgmt->timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->nodeType = nodeType; @@ -1145,7 +1145,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } *qWorkerMgmt = mgmt; @@ -1288,7 +1288,7 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } ctx->level = plan->level; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 1cf4bd9923..59e39e3f6f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -833,7 +833,7 @@ int32_t filterGetRangeRes(void *h, SFilterRange *ra) { if (num == 0) { qError("no range result"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1053,12 +1053,12 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { if (node == NULL) { fltError("empty node"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t type; @@ -1125,7 +1125,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { fltError("invalid right field in unit, operator:%s, rightType:%d", operatorTypeStr(optr), u->right.type); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } } @@ -1208,7 +1208,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { } else { void *data = taosMemoryCalloc(1, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); // reserved space for simple_copy if (NULL == data) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(data, nodesGetValueFromNode(valueNode), tDataTypes[type].bytes); filterAddField(info, NULL, (void **)&data, FLD_TYPE_VALUE, &right, len, true, NULL); @@ -1515,7 +1515,7 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { return DEAL_RES_IGNORE_CHILD; } - ctx->code = TSDB_CODE_QRY_APP_ERROR; + ctx->code = TSDB_CODE_APP_ERROR; fltError("invalid condition type, type:%d", node->condType); @@ -1946,7 +1946,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); if (fi->data == NULL) { fltError("failed to convert in param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); @@ -2014,7 +2014,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0) { qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); @@ -3721,12 +3721,12 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode *pNode) { if (NULL == stat->nodeList) { stat->nodeList = taosArrayInit(10, POINTER_BYTES); if (NULL == stat->nodeList) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(stat->nodeList, &pNode)) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -3847,7 +3847,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); - stat->code = TSDB_CODE_QRY_APP_ERROR; + stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -3908,7 +3908,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); - stat->code = TSDB_CODE_QRY_APP_ERROR; + stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -3994,7 +3994,7 @@ int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) { fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock)); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, id); @@ -4024,14 +4024,14 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) int32_t code = 0; if (pNode == NULL || pInfo == NULL) { fltError("invalid param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } if (*pInfo == NULL) { *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); if (NULL == *pInfo) { fltError("taosMemoryCalloc %d failed", (int32_t)sizeof(SFilterInfo)); - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 55f33b7a3e..2bb38c1678 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -83,7 +83,7 @@ int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockL SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); if (NULL == pLeft) { sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pLeft->numOfRows = pb->info.rows; @@ -104,7 +104,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { sclError("taosHashInit failed, size:%d", 256); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); @@ -162,7 +162,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) { sclError("taosHashPut to set failed"); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } colDataDestroy(out.columnData); @@ -224,7 +224,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { *res = taosMemoryMalloc(pNode->node.resType.bytes); if (NULL == (*res)) { sclError("malloc %d failed", pNode->node.resType.bytes); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); @@ -362,7 +362,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } param->colAlloced = false; break; @@ -370,7 +370,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_COLUMN: { if (NULL == ctx->pBlockList) { sclError("invalid node type for constant calculating, type:%d, src:%p", nodeType(node), ctx->pBlockList); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } SColumnNode *ref = (SColumnNode *)node; @@ -417,7 +417,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } *param = *res; param->colAlloced = false; @@ -459,7 +459,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (pParamList) { @@ -548,7 +548,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SScalarParam *paramList = taosMemoryCalloc(paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } sclSetOperatorValueType(node, ctx); @@ -987,7 +987,7 @@ EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opTy SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -999,7 +999,7 @@ EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opTy SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1126,7 +1126,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1178,7 +1178,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1218,7 +1218,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1270,7 +1270,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1329,7 +1329,7 @@ EDealRes sclWalkFunction(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1346,7 +1346,7 @@ EDealRes sclWalkLogic(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1364,7 +1364,7 @@ EDealRes sclWalkOperator(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1412,7 +1412,7 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); - ctx->code = TSDB_CODE_QRY_APP_ERROR; + ctx->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -1434,7 +1434,7 @@ EDealRes sclWalkCaseWhen(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1485,7 +1485,7 @@ int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); @@ -1603,7 +1603,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); @@ -1613,7 +1613,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } if (1 == res->numOfRows) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 6074c72424..a1995bdf50 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -395,7 +395,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { func = varToTimestamp; } else { sclError("invalid convert outType:%d", pCtx->outType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; @@ -440,7 +440,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); taosMemoryFreeClear(tmp); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } tmp[len] = 0; @@ -642,7 +642,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { } } else { sclError("not supported input type:%d", pCtx->inType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -859,7 +859,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } default: sclError("invalid convert output type:%d", cCtx.outType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 8980d08e89..5e4fe4b8a1 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -63,7 +63,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == pJob->flowCtrl) { SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_SET_JOB_NEED_FLOW_CTRL(pJob); @@ -122,7 +122,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { } SCH_TASK_ELOG("taosHashPut flowCtrl failed, size:%d", (int32_t)sizeof(nctrl)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, @@ -156,13 +156,13 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { ctrl->taskList = taosArrayInit(pLevel->taskNum, POINTER_BYTES); if (NULL == ctrl->taskList) { SCH_TASK_ELOG("taosArrayInit taskList failed, size:%d", (int32_t)pLevel->taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(ctrl->taskList, &pTask)) { SCH_TASK_ELOG("taosArrayPush to taskList failed, size:%d", (int32_t)taosArrayGetSize(ctrl->taskList)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } *enough = false; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 0eb29a3667..af2aeb6461 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -89,34 +89,34 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_INIT) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_INIT: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_EXEC: if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_SUCC: case JOB_TASK_STATUS_FAIL: if (newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; @@ -125,7 +125,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) { @@ -168,7 +168,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { pTask->children = taosArrayInit(childNum, POINTER_BYTES); if (NULL == pTask->children) { SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -182,7 +182,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { if (NULL == taosArrayPush(pTask->children, childTask)) { SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("children info, the %d child TID 0x%" PRIx64, n, (*childTask)->taskId); @@ -197,7 +197,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); if (NULL == pTask->parents) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } else { if (0 != pLevel->level) { @@ -216,7 +216,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { if (NULL == taosArrayPush(pTask->parents, parentTask)) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); @@ -278,13 +278,13 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pJob->levels = taosArrayInit(levelNum, sizeof(SSchLevel)); if (NULL == pJob->levels) { SCH_JOB_ELOG("taosArrayInit %d failed", levelNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pJob->levelNum = levelNum; @@ -300,7 +300,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t i = 0; i < levelNum; ++i) { if (NULL == taosArrayPush(pJob->levels, &level)) { SCH_JOB_ELOG("taosArrayPush level failed, level:%d", i); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pLevel = taosArrayGet(pJob->levels, i); @@ -323,7 +323,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } for (int32_t n = 0; n < taskNum; ++n) { @@ -335,7 +335,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchTask *pTask = taosArrayPush(pLevel->subTasks, &task); if (NULL == pTask) { SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schInitTask(pJob, pTask, plan, pLevel)); @@ -344,12 +344,12 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &pTask, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } if (0 != taosHashPut(pJob->taskList, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to taskList failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } ++pJob->taskNum; @@ -702,7 +702,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pJob->attr.explainMode = pReq->pDag->explainInfo.mode; @@ -728,7 +728,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { HASH_ENTRY_LOCK); if (NULL == pJob->taskList) { SCH_JOB_ELOG("taosHashInit %d taskList failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schValidateAndBuildJob(pReq->pDag, pJob)); @@ -741,7 +741,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } tsem_init(&pJob->rspSem, 0, 0); @@ -865,8 +865,8 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + schDirectPostJobRes(pReq, TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); @@ -879,8 +879,8 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + schDirectPostJobRes(pReq, TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); @@ -894,7 +894,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (!SCH_JOB_NEED_FETCH(pJob)) { SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } if (status != JOB_TASK_STATUS_PART_SUCC) { @@ -911,7 +911,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq return TSDB_CODE_SUCCESS; default: SCH_JOB_ELOG("unknown operation type %d", type); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } if (schJobNeedToStop(pJob, &status)) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a6d7df349..fa0e7ff554 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -371,7 +371,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SExplainRsp rsp = {0}; if (tDeserializeSExplainRsp(msg, msgSize, &rsp)) { tFreeSExplainRsp(&rsp); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schProcessExplainRsp(pJob, pTask, &rsp)); @@ -506,7 +506,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->queryId = pJob->queryId; @@ -523,7 +523,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->head.isHbParam = true; @@ -543,7 +543,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { qError("calloc SSchTaskCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->pTrans = trans->pTrans; @@ -558,7 +558,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } msgSendInfo->paramFreeFp = taosMemoryFree; @@ -621,7 +621,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { break; default: qError("unknown msg type for callback, msgType:%d", msgType); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } return TSDB_CODE_SUCCESS; @@ -632,7 +632,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->head.isHbParam = true; @@ -662,7 +662,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pDst->args) { qError("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SRpcCtxVal dst = {0}; @@ -679,7 +679,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { qError("taosHashPut msg %d to rpcCtx failed", *msgType); (*pSrc->freeFunc)(dst.val); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pIter = taosHashIterate(pSrc->args, pIter); @@ -706,19 +706,19 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int32_t msgType = TDMT_SCH_QUERY_HEARTBEAT_RSP; @@ -735,7 +735,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, true)); @@ -780,7 +780,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; @@ -790,7 +790,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SRpcCtxVal ctxVal = {.val = pExplainMsgSendInfo, .clone = schCloneSMsgSendInfo}; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, false)); @@ -815,7 +815,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); if (NULL == dst) { qError("malloc SSchHbCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(dst, pSrc, sizeof(*dst)); @@ -827,7 +827,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SSchTaskCallbackParam *dst = taosMemoryMalloc(sizeof(SSchTaskCallbackParam)); if (NULL == dst) { qError("malloc SSchTaskCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(dst, pSrc, sizeof(*dst)); @@ -842,7 +842,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == pDst) { qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pDst, pSrc, sizeof(*pSrc)); @@ -959,17 +959,17 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { qError("calloc hb req %d failed", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int64_t transporterId = 0; @@ -1016,7 +1016,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(msg, pTask->msg, msgSize); @@ -1037,7 +1037,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } tSerializeSVDeleteReq(msg, msgSize, &req); @@ -1067,19 +1067,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSSubQueryMsg(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSubQueryMsg(msg, msgSize, &qMsg) < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg failed, msgSize:%d", msgSize); taosMemoryFree(msg); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } persistHandle = true; @@ -1098,18 +1098,18 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSResFetchReq get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { SCH_TASK_ELOG("tSerializeSResFetchReq %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -1126,19 +1126,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); taosMemoryFree(msg); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -1154,16 +1154,16 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_JOB_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } persistHandle = true; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c8b8db367f..0f15002d9b 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -73,7 +73,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * taosHashInit(pTask->maxExecTimes, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); @@ -112,7 +112,7 @@ int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad if (taosHashPut(pTask->execNodes, &execId, sizeof(execId), &nodeInfo, sizeof(nodeInfo))) { SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task execNode added, execId:%d, handle:%p", execId, nodeInfo.handle); @@ -530,7 +530,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { } SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task added to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); @@ -555,7 +555,7 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -582,7 +582,7 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -607,7 +607,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -712,7 +712,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, @@ -740,13 +740,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (pTask->plan->execNode.epSet.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("use execNode in plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); @@ -756,7 +756,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); @@ -1173,7 +1173,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { SCH_TASK_ELOG("taosMemoryMalloc %d failed", (int)sizeof(SSchTimerParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->rId = pJob->refId; @@ -1184,7 +1184,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index a5a15cd1c6..0582184730 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -235,7 +235,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { if (NULL == hb) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_LOCK(SCH_WRITE, &hb->lock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 91c3b5d7a1..1926c26dc9 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -39,19 +39,19 @@ int32_t schedulerInit() { schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } schMgmt.hbConnections = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == schMgmt.hbConnections) { qError("taosHashInit hb connections failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } schMgmt.timer = taosTmrInit(0, 0, 0, "scheduler"); if (NULL == schMgmt.timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index cbcf111a85..f51fe344df 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -41,7 +41,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->pCache = taosLRUCacheInit(30 * 1024 * 1024, 1, .5); if (pLogStore->pCache == NULL) { taosMemoryFree(pLogStore); - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index d77acbbb6f..64dde15e76 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -105,7 +105,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { void* ptr = taosMemoryRealloc(buf, capacity); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } buf = ptr; @@ -632,7 +632,7 @@ int walRollFileInfo(SWal* pWal) { // TODO: change to emplace back SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pNewInfo->firstVer = pWal->vers.lastVer + 1; @@ -664,7 +664,7 @@ char* walMetaSerialize(SWal* pWal) { if (pFiles) { cJSON_Delete(pFiles); } - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); @@ -895,7 +895,7 @@ int walLoadMeta(SWal* pWal) { int size = (int)fileSize; char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } memset(buf, 0, size + 5); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 5d0f020b02..a683ceb2b1 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -168,7 +168,7 @@ _err: } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { - if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR; + if (pWal == NULL) return TSDB_CODE_APP_ERROR; if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1350ca0c37..76266d338b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -48,7 +48,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pReader->pHead = taosMemoryMalloc(sizeof(SWalCkHead)); if (pReader->pHead == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pReader); return NULL; } @@ -280,7 +280,7 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pRead->pHead = ptr; @@ -437,7 +437,7 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(*ppHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } *ppHead = ptr; @@ -546,7 +546,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 18eeca030e..1cea9b8e19 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -109,11 +109,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, "Invalid password") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, "Database name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, "Invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, "System out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, "Query cache erased") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, "Query terminated") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, "Result set too large to be sorted") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, "Application error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, "No write permission") @@ -142,7 +140,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") @@ -157,7 +154,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_DISABLED, "User is disabled") // mnode-sdb TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE, "Object already there") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_APP_ERROR, "Unexpected generic error in sdb") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_TABLE_TYPE, "Invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_NOT_THERE, "Object not there") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_KEY_TYPE, "Invalid key type") @@ -335,8 +331,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists" TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, "No permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, "Data file(s) corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, "Tag too old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, "Timestamp data out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, "Submit message is messed up") @@ -360,8 +354,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, "Invalid message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, "Unexpected generic error in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, "Duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") @@ -426,7 +418,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INVALID_CONFIG, "TQ invalid config") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INIT_FAILED, "TQ init falied") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_DISK_PERMISSIONS, "TQ no disk permissions") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_CORRUPTED, "TQ file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TQ_OUT_OF_MEMORY, "TQ out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_ALREADY_EXISTS, "TQ file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FAILED_TO_CREATE_DIR, "TQ failed to create dir") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_META_NO_SUCH_KEY, "TQ meta no such key") @@ -437,11 +428,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") @@ -455,7 +444,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, "tfs no mount at tier" TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") // catalog TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog internal error") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f62c3f7f58..28ba7ea60f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -113,7 +113,6 @@ ,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/interp.sim -#,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim From 658cb393a775207824a663fc87415442252620f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:11:20 +0800 Subject: [PATCH 102/120] enh(query): improve the performance and add some todo --- source/libs/executor/src/executil.c | 1 + source/libs/scalar/src/scalar.c | 5 +---- source/util/src/tpagedbuf.c | 12 ++++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1241bdf5a4..b9681a4fde 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -129,6 +129,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in void* pData = NULL; pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + // todo avoid repeated malloc memory size_t keyLen = 0; int32_t iter = 0; while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 55f33b7a3e..b60b651ebc 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -165,10 +165,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - colDataDestroy(out.columnData); - taosMemoryFreeClear(out.columnData); - out.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); - + colInfoDataCleanup(out.columnData, out.numOfRows); cell = cell->pNext; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0f4c51060b..ad08b15c25 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -30,6 +30,7 @@ struct SDiskbasedBuf { TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path + char* prefix; // file name prefix int32_t pageSize; // current used page size int32_t inMemPages; // numOfPages that are allocated in memory SList* freePgList; // free page list @@ -48,6 +49,12 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { + if (pBuf->path[0] == '\0') { // prepare the file name when needed it + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); + pBuf->path = taosMemoryStrDup(path); + } + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (pBuf->pFile == NULL) { @@ -356,10 +363,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "paged-buf", path); - pPBuf->path = strdup(path); + pPBuf->prefix = (char*) dir; pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); From b6199d8d99f9c527d027af8ffe1552e913e762ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:37:42 +0800 Subject: [PATCH 103/120] enh(query): update the aligned bits for colInfodata --- source/common/src/tdatablock.c | 2 +- source/os/src/osMemory.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c53e771f42..c253e842d6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 32 +#define MALLOC_ALIGN_BYTES 256 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2f30e8977a..230ca5b968 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -351,7 +351,8 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { ASSERT(0); #else #if defined(LINUX) - return memalign(alignment, size); + void* p = memalign(alignment, size); + return p; #else return taosMemoryMalloc(size); #endif From 85c1e3aabf9a8e6c04eff24ea4bcaa5a1410f0f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:41:56 +0800 Subject: [PATCH 104/120] refactor: add an assert. --- source/os/src/osMemory.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 230ca5b968..b2d1dcf58e 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -352,6 +352,8 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void* p = memalign(alignment, size); + ASSERT((((uint64_t)p) & (alignment - 1)) == 0x0); + return p; #else return taosMemoryMalloc(size); From fa43f86fe1af85ccc050c3ecb6232c548ef2a7fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:43:03 +0800 Subject: [PATCH 105/120] refactor: add an assert. --- source/common/src/tdatablock.c | 3 ++- source/os/src/osMemory.c | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c253e842d6..2832e6d16e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1191,7 +1191,6 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); - ASSERT(pColumn->info.bytes); // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned @@ -1207,6 +1206,8 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); + if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index b2d1dcf58e..230ca5b968 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -352,8 +352,6 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void* p = memalign(alignment, size); - ASSERT((((uint64_t)p) & (alignment - 1)) == 0x0); - return p; #else return taosMemoryMalloc(size); From c3267d5294357bdcc2bb20c2c6eb7efe8fd0a45f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 3 Dec 2022 15:08:45 +0800 Subject: [PATCH 106/120] fix: fix db release issue --- source/libs/catalog/src/ctgCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 6d28420f79..f41058584f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -440,7 +440,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); return TSDB_CODE_SUCCESS; @@ -520,7 +520,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } From a3200b555cc992bcd06a7d45f93e89a0d0201d72 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 17:03:31 +0800 Subject: [PATCH 107/120] fix(query): fix null ptr access. --- source/util/src/tpagedbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index ad08b15c25..37874ae250 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -49,7 +49,7 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - if (pBuf->path[0] == '\0') { // prepare the file name when needed it + if (pBuf->path == NULL) { // prepare the file name when needed it char path[PATH_MAX] = {0}; taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); pBuf->path = taosMemoryStrDup(path); From 0a219ec88ff30f7eb87faa7bda28b8a14040fe6e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 3 Dec 2022 17:38:38 +0800 Subject: [PATCH 108/120] fix(case): taos shell modify Query OK output string --- tests/system-test/0-others/taosShell.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index e3095e8b93..bcf27800f2 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -62,7 +62,7 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key print ('taos login success! Here can run sql, taos> ') if len(sqlString) != 0: child.sendline (sqlString) - w = child.expect(["Query OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) + w = child.expect(["Query OK", "Create OK", "Insert OK", "Drop OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) if w == 0: return "TAOS_OK" else: @@ -233,7 +233,7 @@ class TDTestCase: tdLog.printNoPrefix("================================ parameter: -s") newDbName="dbss" keyDict['s'] = "\"create database " + newDbName + "\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s fail") @@ -246,17 +246,17 @@ class TDTestCase: tdLog.exit("create db fail after taos -s %s fail"%(keyDict['s'])) keyDict['s'] = "\"create table " + newDbName + ".stb (ts timestamp, c int) tags (t int)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s create table fail") keyDict['s'] = "\"create table " + newDbName + ".ctb0 using " + newDbName + ".stb tags (0) " + newDbName + ".ctb1 using " + newDbName + ".stb tags (1)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s create table fail") keyDict['s'] = "\"insert into " + newDbName + ".ctb0 values('2021-04-01 08:00:00.000', 10)('2021-04-01 08:00:01.000', 20) " + newDbName + ".ctb1 values('2021-04-01 08:00:00.000', 11)('2021-04-01 08:00:01.000', 21)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s insert data fail") @@ -401,27 +401,27 @@ class TDTestCase: tdLog.printNoPrefix("================================ parameter: -w") newDbName="dbw" keyDict['s'] = "\"create database " + newDbName + "\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w fail") keyDict['s'] = "\"create table " + newDbName + ".ntb (ts timestamp, c binary(128))\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w create table fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.001', 'abcd0123456789')('2021-04-01 08:00:00.002', 'abcd012345678901234567890123456789') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.003', 'aaaaaaaaaaaaaaaaaaaa')('2021-04-01 08:00:01.004', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.005', 'cccccccccccccccccccc')('2021-04-01 08:00:01.006', 'dddddddddddddddddddddddddddddddddddddddd') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") From a976d3a4c10ed6fdad29976cdb8c9b713764af49 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 3 Dec 2022 17:48:47 +0800 Subject: [PATCH 109/120] fix(case): pexpect module expect return index --- tests/system-test/0-others/taosShell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index bcf27800f2..5c7bb0443a 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -63,7 +63,7 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key if len(sqlString) != 0: child.sendline (sqlString) w = child.expect(["Query OK", "Create OK", "Insert OK", "Drop OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) - if w == 0: + if w == 0 or w == 1 or w == 2: return "TAOS_OK" else: print(1) From 392fe988a0f4de1726475be1568242d28864d8ba Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 20:16:56 +0800 Subject: [PATCH 110/120] fix(query): remove invalid assert for mac&win. --- source/common/src/tdatablock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2832e6d16e..dfd0b68039 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1206,7 +1206,11 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + + // todo remove it soon +#if defined LINUX ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); +#endif if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); From a7807d3ee4a08cd98c16779b7569a632c5340c41 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 3 Dec 2022 20:53:41 +0800 Subject: [PATCH 111/120] fix: synchronize syncPropose for executing blocking msgs in vnodeProposeMsg --- include/common/tmsg.h | 4 +++ include/util/taoserror.h | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/dnode/vnode/src/vnd/vnodeSync.c | 45 +++++++++++++++++++------- source/libs/sync/src/syncMain.c | 6 +++- source/libs/sync/src/syncPipeline.c | 9 ++++-- source/util/src/terror.c | 1 + 7 files changed, 54 insertions(+), 15 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0a082a37e4..f554989edc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -66,6 +66,10 @@ extern int32_t tMsgDict[]; typedef uint16_t tmsg_t; +static inline bool vnodeIsMsgBlock(tmsg_t type) { + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || + (type == TDMT_VND_UPDATE_TAG_VAL); +} /* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 490dbf7744..e900eff3fd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -381,6 +381,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527) #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) +#define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fc6b126cc5..ee2e8f6f6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -181,14 +181,15 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); + terrno = TSDB_CODE_VND_DUP_REQUEST; pRsp->info.handle = NULL; return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); - ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); + ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); pVnode->state.applied = version; pVnode->state.applyTerm = pMsg->info.conn.applyTerm; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 20ae16ae43..1c2139c95d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -18,14 +18,15 @@ #define BATCH_ENABLE 0 -static inline bool vnodeIsMsgBlock(tmsg_t type) { - return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || - (type == TDMT_VND_UPDATE_TAG_VAL); -} - static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); + tsem_wait(&pVnode->syncSem); +} + +static inline void vnodeWaitBlockMsgOld(SVnode *pVnode, const SRpcMsg *pMsg) { if (vnodeIsMsgBlock(pMsg->msgType)) { const STraceId *trace = &pMsg->info.traceId; taosThreadMutexLock(&pVnode->lock); @@ -115,21 +116,31 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) { if (*arrSize <= 0) return; + SRpcMsg *pLastMsg = pMsgArr[*arrSize - 1]; + taosThreadMutexLock(&pVnode->lock); int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize); + bool wait = (code == 0 && vnodeIsBlockMsg(pLastMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + } + taosThreadMutexUnlock(&pVnode->lock); + if (code > 0) { for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleWriteMsg(pVnode, pMsgArr[i]); } - } else if (code == 0) { - vnodeWaitBlockMsg(pVnode, pMsgArr[*arrSize - 1]); - } else { + } else if (code < 0) { if (terrno != 0) code = terrno; for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleProposeError(pVnode, pMsgArr[i], code); } } + if (wait) vnodeWaitBlockMsg(pVnode, pLastMsg); + pLastMsg = NULL; + for (int32_t i = 0; i < *arrSize; ++i) { SRpcMsg *pMsg = pMsgArr[i]; const STraceId *trace = &pMsg->info.traceId; @@ -206,16 +217,23 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) #else static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { + taosThreadMutexLock(&pVnode->lock); int32_t code = syncPropose(pVnode->sync, pMsg, isWeak); + bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + } + taosThreadMutexUnlock(&pVnode->lock); + if (code > 0) { vnodeHandleWriteMsg(pVnode, pMsg); - } else if (code == 0) { - vnodeWaitBlockMsg(pVnode, pMsg); - } else { + } else if (code < 0) { if (terrno != 0) code = terrno; vnodeHandleProposeError(pVnode, pMsg, code); } + if (wait) vnodeWaitBlockMsg(pVnode, pMsg); return code; } @@ -274,6 +292,11 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64, vgId, pMsg, TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex); + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, blocking msg obtained from apply-queue. index:%" PRId64 ", term: %" PRId64 ", type: %s", vgId, + pMsg->info.conn.applyIndex, pMsg->info.conn.applyTerm, TMSG_INFO(pMsg->msgType)); + } + SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; if (rsp.code == 0) { if (vnodeProcessWriteMsg(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index abfd8043c9..1fe2af696c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2638,7 +2638,11 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn (*pRetIndex) = index; } - return syncNodeAppend(ths, pEntry); + int32_t code = syncNodeAppend(ths, pEntry); + if (code < 0 && ths->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { + ASSERT(false && "failed to append blocking msg"); + } + return code; } return -1; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index d5d77274a9..74a19608dc 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "syncPipeline.h" +#include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" #include "syncRaftEntry.h" @@ -442,6 +443,11 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn return 0; } + if (pNode->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { + sTrace("vgId:%d, blocking msg ready to execute. index:%" PRId64 ", term: %" PRId64 ", type: %s", pNode->vgId, + pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); + } + SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); @@ -514,9 +520,8 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } continue; } - if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { - sError("vgId:%d, failed to commit sync log entry. index:%" PRId64 ", term:%" PRId64 + sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role: %d, current term: %" PRId64, vgId, pEntry->index, pEntry->term, role, term); goto _out; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 18eeca030e..5d8a18fc46 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -326,6 +326,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_NOT_EXISTS, "Table column not exis TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subscribed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_DUP_REQUEST, "Duplicate write request") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") From 803762c4e0573e2488bea350859a6413a8af957b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 22:57:00 +0800 Subject: [PATCH 112/120] enh(query): avoid repeatly load del file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 65 ++++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 3 +- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d9f0c55d24..25b6fc2a9a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -170,7 +170,9 @@ struct STsdbReader { SIOCostSummary cost; STSchema* pSchema; // the newest version schema STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; SVersionRange verRange; SBlockInfoBuf blockInfoBuf; int32_t step; @@ -2531,42 +2533,17 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* } int32_t code = 0; - STsdb* pTsdb = pReader->pTsdb; - SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; - if (pDelFile) { - SDelFReader* pDelFReader = NULL; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - SArray* aDelIdx = taosArrayInit(4, sizeof(SDelIdx)); - if (aDelIdx == NULL) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - // TODO: opt the perf of read del index - code = tsdbReadDelIdx(pDelFReader, aDelIdx); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(aDelIdx); - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - + if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { SDelIdx idx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - SDelIdx* pIdx = taosArraySearch(aDelIdx, &idx, tCmprDelIdx, TD_EQ); + SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pDelFReader, pIdx, pDelData); + code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData); } - taosArrayDestroy(aDelIdx); - tsdbDelFReaderClose(&pDelFReader); - if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -2662,6 +2639,27 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { } taosArrayDestroy(pIndexList); + + SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; + if (pReader->pDelFReader == NULL && pDelFile != NULL) { + int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); + if (pReader->pDelIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pReader->pDelIdx); + return code; + } + } + return TSDB_CODE_SUCCESS; } @@ -3912,6 +3910,15 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } + if (pReader->pDelFReader != NULL) { + tsdbDelFReaderClose(&pReader->pDelFReader); + } + + if (pReader->pDelIdx != NULL) { + taosArrayDestroy(pReader->pDelIdx); + pReader->pDelIdx = NULL; + } + tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 294a4bd3e4..1d09bd4936 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1477,9 +1477,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { } taosMemoryFree(pReader); } - *ppReader = NULL; -_exit: + *ppReader = NULL; return code; } From fca20668f56fe712b4806ce3414d3bffef2a9248 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 23:52:38 +0800 Subject: [PATCH 113/120] fix(query): check for 0 tables in tsdbreader --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 25b6fc2a9a..fece5df28e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3960,6 +3960,9 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { blockDataCleanup(pBlock); SReaderStatus* pStatus = &pReader->status; + if (taosHashGetSize(pStatus->pTableMap) == 0){ + return false; + } if (pStatus->loadFromFile) { int32_t code = buildBlockFromFiles(pReader); @@ -3977,8 +3980,6 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { buildBlockFromBufferSequentially(pReader); return pBlock->info.rows > 0; } - - return false; } bool tsdbNextDataBlock(STsdbReader* pReader) { From 21ff35ee421dde49efe4df311d3fc9961e559251 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 4 Dec 2022 08:13:49 +0800 Subject: [PATCH 114/120] fix: truncate WAL on need while syncLogBufferRollback --- source/libs/sync/src/syncPipeline.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 74a19608dc..5d6905f99e 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -984,6 +984,7 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex ", %" PRId64 ")", pNode->vgId, toIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + // trunc buffer SyncIndex index = pBuf->endIndex - 1; while (index >= toIndex) { SSyncRaftEntry* pEntry = pBuf->entries[index % pBuf->size].pItem; @@ -997,6 +998,17 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex pBuf->endIndex = toIndex; pBuf->matchIndex = TMIN(pBuf->matchIndex, index); ASSERT(index + 1 == toIndex); + + // trunc wal + SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); + if (lastVer >= toIndex && pNode->pLogStore->syncLogTruncate(pNode->pLogStore, toIndex) < 0) { + sError("vgId:%d, failed to truncate log store since %s. from index:%" PRId64 "", pNode->vgId, terrstr(), toIndex); + return -1; + } + lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); + ASSERT(toIndex == lastVer + 1); + + syncLogBufferValidate(pBuf); return 0; } From 575c5e89fc0bd5d61567bac561e15cc3cc054846 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 4 Dec 2022 09:49:27 +0800 Subject: [PATCH 115/120] refactor(sync): add config item in sync --- include/common/tglobal.h | 13 +++++++++---- include/libs/sync/sync.h | 2 +- source/common/src/tglobal.c | 25 ++++++++++++++++++++----- source/libs/sync/src/syncMain.c | 9 ++++++--- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 005cf36d5e..92672311d0 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -64,6 +64,11 @@ extern int32_t tsNumOfSnodeStreamThreads; extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsRpcQueueMemoryAllowed; +// sync raft +extern int32_t tsElectInterval; +extern int32_t tsHeartbeatInterval; +extern int32_t tsHeartbeatTimeout; + // monitor extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; @@ -126,9 +131,9 @@ extern char tsUdfdResFuncs[]; extern char tsUdfdLdLibPath[]; // schemaless -extern char tsSmlChildTableName[]; -extern char tsSmlTagName[]; -extern bool tsSmlDataFormat; +extern char tsSmlChildTableName[]; +extern char tsSmlTagName[]; +extern bool tsSmlDataFormat; extern int32_t tsSmlBatchSize; // wal @@ -146,7 +151,7 @@ extern int32_t tsUptimeInterval; extern int32_t tsRpcRetryLimit; extern int32_t tsRpcRetryInterval; -//#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) +// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 1fa862b145..7f9b240f63 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -43,7 +43,7 @@ extern "C" { #define SYNC_MAX_RETRY_BACKOFF 5 #define SYNC_LOG_REPL_RETRY_WAIT_MS 100 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 -#define SYNC_HEART_TIMEOUT_MS 1000 * 8 +#define SYNC_HEART_TIMEOUT_MS 1000 * 15 #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2e8cb5e5f7..3bcfddb8b2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -55,6 +55,11 @@ int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; +// sync raft +int32_t tsElectInterval = 25 * 1000; +int32_t tsHeartbeatInterval = 1000; +int32_t tsHeartbeatTimeout = 20 * 1000; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; @@ -74,8 +79,8 @@ char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) -bool tsSmlDataFormat = false; -int32_t tsSmlBatchSize = 10000; +bool tsSmlDataFormat = false; +int32_t tsSmlBatchSize = 10000; // query int32_t tsQueryPolicy = 1; @@ -198,9 +203,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg); #endif -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -423,6 +426,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; @@ -728,6 +735,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; + tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; + tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; + tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; @@ -737,6 +748,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsWalFsyncDataSizeLimit = cfgGetItem(pCfg, "walFsyncDataSizeLimit")->i64; + tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; + tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; + tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath)); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 1fe2af696c..a00b639a9c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -22,8 +22,8 @@ #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" -#include "syncPipeline.h" #include "syncMessage.h" +#include "syncPipeline.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" @@ -35,6 +35,7 @@ #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "tglobal.h" #include "tref.h" static void syncNodeEqPingTimer(void* param, void* tmrId); @@ -1115,7 +1116,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->hbrSlowNum = 0; pSyncNode->tmrRoutineNum = 0; - sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); + sNInfo(pSyncNode, "sync open, node:%p", pSyncNode); + sTrace("vgId:%d, tsElectInterval:%d, tsHeartbeatInterval:%d, tsHeartbeatTimeout:%d", pSyncNode->vgId, tsElectInterval, + tsHeartbeatInterval, tsHeartbeatTimeout); return pSyncNode; @@ -1229,7 +1232,7 @@ void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return; - sNTrace(pSyncNode, "sync close, data:%p", pSyncNode); + sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); int32_t ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); From 0d687a533efbad0c51cad17eabbf37efd6f5023a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 4 Dec 2022 11:35:54 +0800 Subject: [PATCH 116/120] fix(query): check for null ptr. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 34 ++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fece5df28e..7f513d2b2d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2534,6 +2534,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* int32_t code = 0; SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); + ASSERT(pReader->pReadSnap != NULL); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { @@ -2640,23 +2641,26 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pIndexList); - SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; - if (pReader->pDelFReader == NULL && pDelFile != NULL) { - int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pReadSnap != NULL) { - pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); - if (pReader->pDelIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; + if (pReader->pDelFReader == NULL && pDelFile != NULL) { + int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pReader->pDelIdx); - return code; + pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); + if (pReader->pDelIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pReader->pDelIdx); + return code; + } } } From 69850a1a2b1e011130133f16866f248261c7f53d Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 12:28:36 +0800 Subject: [PATCH 117/120] fix: dataDir process of multi-tier storage conform to 2.6 --- source/util/src/tconfig.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 586586f742..cb47838b1b 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -294,8 +294,8 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, SDiskCfg cfg = {0}; tstrncpy(cfg.dir, value, sizeof(cfg.dir)); - cfg.level = atoi(level); - cfg.primary = atoi(primary); + cfg.level = level ? atoi(level) : 0; + cfg.primary = primary ? atoi(primary) : 1; void *ret = taosArrayPush(pItem->array, &cfg); if (ret == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -660,13 +660,12 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } uInfo("load from env variables cfg success"); @@ -703,13 +702,12 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } uInfo("load from env cmd cfg success"); @@ -768,13 +766,12 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } taosCloseFile(&pFile); @@ -828,13 +825,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } taosCloseFile(&pFile); @@ -993,13 +989,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { paGetToken(value2 + vlen2 + 1, &value3, &vlen3); if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } tjsonDelete(pJson); From 4dbc6075a381dae7e0c009afa3d22802c54f1d49 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 12:32:08 +0800 Subject: [PATCH 118/120] fix: dataDir process of multi-tier storage conform to 2.6 --- source/util/src/tconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index cb47838b1b..3d05f80acf 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -891,7 +891,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { // code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); // if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { +// if (strcasecmp(name, "dataDir") == 0) { // code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); // if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; // } From 1e15e23ccefca0c13bab7507bbddec328f20ccb9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 4 Dec 2022 15:30:06 +0800 Subject: [PATCH 119/120] feat: taosbenchmark supports vgid (#18657) * test: update taos-tools e0f2fe3 * feat: taosbenchmark supports vgid * fix: util func refactor * test: update taos-tools 09372db * test: update taos-tools e62c5ea * test: comment off replica line for ci * test: comment off invalid_commandline.py and demo.py --- cmake/taostools_CMakeLists.txt.in | 2 +- tests/develop-test/5-taos-tools/taosbenchmark/demo.py | 2 +- .../5-taos-tools/taosbenchmark/invalid_commandline.py | 2 +- tests/parallel_test/cases.task | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index db727907b7..2d7bcf6592 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 823fae5 + GIT_TAG e62c5ea SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index 6be5117b08..25a8d92f7e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -28,7 +28,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py index 73894d5e33..e5a2551e63 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py @@ -25,7 +25,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 28ba7ea60f..c90b2499a4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1022,9 +1022,9 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py From feceb0f1f45ca8a9e3b56a80a2334a7b9061f026 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 15:31:08 +0800 Subject: [PATCH 120/120] fix: multi-tier storage test case update --- tests/system-test/1-insert/mutil_stage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 3e2bec130e..8780ecc477 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -147,19 +147,20 @@ class TDTestCase: def __current_cfg(self): cfg_list = [] current_case1 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" ] - current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + #current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + current_case2 = [] for i in range(9): current_case2.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") # TD-17773bug current_case3 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", + #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE1}0 {L1} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}0 {L2} {NON_PRIMARY_DIR}",