From 82ad690eb4b9b57f9893fee98ed604f5713355f9 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 13 Nov 2023 20:20:45 +0800 Subject: [PATCH 01/43] rewrite nodesNodeName --- include/common/tmsg.h | 3 +- include/libs/nodes/nodes.h | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 567 +++++++++---------------- source/libs/nodes/src/nodesUtilFuncs.c | 1 + 4 files changed, 200 insertions(+), 372 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 86d34502c6..4d4744999f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -414,7 +414,8 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, - QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL + QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, + QUERY_NODE_END } ENodeType; typedef struct { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 7fbdbfb211..21d33249b8 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -101,6 +101,7 @@ typedef struct SNodeList { typedef struct SNodeAllocator SNodeAllocator; +void initNodeName(); int32_t nodesInitAllocatorSet(); void nodesDestroyAllocatorSet(); int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f3087dd5d4..582ecf6753 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -27,378 +27,203 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj); static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** pNode); static int32_t makeNodeByJson(const SJson* pJson, SNode** pNode); + +typedef const char* taoscstr; +taoscstr nodeNames[QUERY_NODE_END] = {}; + +#define TAOS_DEFINE_NODE_NAME(type, msg) nodeNames[type] = msg; + +void initNodeName() { + TAOS_DEFINE_NODE_NAME(QUERY_NODE_COLUMN, "Column") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_VALUE, "Value") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_OPERATOR, "Operator") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_CONDITION, "LogicCondition") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_FUNCTION, "Function") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_REAL_TABLE, "RealTable") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_TEMP_TABLE, "TempTable") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_JOIN_TABLE, "JoinTable") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_GROUPING_SET, "GroupingSet") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ORDER_BY_EXPR, "OrderByExpr") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LIMIT, "Limit") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_STATE_WINDOW, "StateWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SESSION_WINDOW, "SessionWinow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_INTERVAL_WINDOW, "IntervalWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_NODE_LIST, "NodeList") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_FILL, "Fill") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_RAW_EXPR, "RawExpr") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_TARGET, "Target") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DATABLOCK_DESC, "DataBlockDesc") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SLOT_DESC, "SlotDesc") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_COLUMN_DEF, "ColumnDef") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DOWNSTREAM_SOURCE, "DownstreamSource") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DATABASE_OPTIONS, "DatabaseOptions") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_TABLE_OPTIONS, "TableOptions") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_INDEX_OPTIONS, "IndexOptions") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_EXPLAIN_OPTIONS, "ExplainOptions") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_STREAM_OPTIONS, "StreamOptions") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LEFT_VALUE, "LeftValue") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_WHEN_THEN, "WhenThen") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CASE_WHEN, "CaseWhen") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_EVENT_WINDOW, "EventWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SET_OPERATOR, "SetOperator") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SELECT_STMT, "SelectStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_VNODE_MODIFY_STMT, "VnodeModifStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_DATABASE_STMT, "CreateDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_DATABASE_STMT, "DropDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_DATABASE_STMT, "AlterDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_FLUSH_DATABASE_STMT, "FlushDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_TRIM_DATABASE_STMT, "TrimDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_TABLE_STMT, "CreateTableStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_SUBTABLE_CLAUSE, "CreateSubtableClause") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_MULTI_TABLES_STMT, "CreateMultiTableStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_TABLE_CLAUSE, "DropTableClause") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_TABLE_STMT, "DropTableStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_SUPER_TABLE_STMT, "DropSuperTableStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_TABLE_STMT, "AlterTableStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_SUPER_TABLE_STMT, "AlterSuperTableStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_USER_STMT, "CreateUserStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_USER_STMT, "AlterUserStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_USER_STMT, "DropUserStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_USE_DATABASE_STMT, "UseDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_DNODE_STMT, "CreateDnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_DNODE_STMT, "DropDnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_DNODE_STMT, "AlterDnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_INDEX_STMT, "CreateIndexStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_INDEX_STMT, "DropIndexStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_QNODE_STMT, "CreateQnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_QNODE_STMT, "DropQnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_SNODE_STMT, "CreateSnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_SNODE_STMT, "DropSnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_MNODE_STMT, "CreateMnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_MNODE_STMT, "DropMnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_TOPIC_STMT, "CreateTopicStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_TOPIC_STMT, "DropTopicStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_CGROUP_STMT, "DropConsumerGroupStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_LOCAL_STMT, "AlterLocalStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_EXPLAIN_STMT, "ExplainStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DESCRIBE_STMT, "DescribeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_COMPACT_DATABASE_STMT, "CompactDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_STREAM_STMT, "CreateStreamStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_STREAM_STMT, "DropStreamStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PAUSE_STREAM_STMT, "PauseStreamStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESUME_STREAM_STMT, "ResumeStreamStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_BALANCE_VGROUP_STMT, "BalanceVgroupStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, "BalanceVgroupLeaderStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_MERGE_VGROUP_STMT, "MergeVgroupStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DB_ALIVE_STMT, "ShowDbAliveStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, "ShowClusterAliveStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT, "RedistributeVgroupStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SPLIT_VGROUP_STMT, "SplitVgroupStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_GRANT_STMT, "GrantStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_REVOKE_STMT, "RevokeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DNODES_STMT, "ShowDnodesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_MNODES_STMT, "ShowMnodesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_MODULES_STMT, "ShowModulesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_QNODES_STMT, "ShowQnodesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_SNODES_STMT, "ShowSnodesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_BNODES_STMT, "ShowBnodesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CLUSTER_STMT, "ShowClusterStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DATABASES_STMT, "ShowDatabaseStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_FUNCTIONS_STMT, "ShowFunctionsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_INDEXES_STMT, "ShowIndexesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_STABLES_STMT, "ShowStablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_STREAMS_STMT, "ShowStreamsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TABLES_STMT, "ShowTablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TAGS_STMT, "ShowTagsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_USERS_STMT, "ShowUsersStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_LICENCES_STMT, "ShowGrantsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_VGROUPS_STMT, "ShowVgroupsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TOPICS_STMT, "ShowTopicsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CONSUMERS_STMT, "ShowConsumersStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_QUERIES_STMT, "ShowQueriesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_VARIABLES_STMT, "ShowVariablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DNODE_VARIABLES_STMT, "ShowDnodeVariablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TRANSACTIONS_STMT, "ShowTransactionsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, "ShowSubscriptionsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_VNODES_STMT, "ShowVnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, "ShowUserPrivilegesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_DATABASE_STMT, "ShowCreateDatabasesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_TABLE_STMT, "ShowCreateTablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_STABLE_STMT, "ShowCreateStablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_VIEW_STMT, "ShowCreateViewStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, "ShowTableDistributedStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, "ShowLocalVariablesStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TABLE_TAGS_STMT, "ShowTableTagsStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DELETE_STMT, "DeleteStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_INSERT_STMT, "InsertStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_DNODE_STMT, "RestoreDnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_QNODE_STMT, "RestoreQnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_MNODE_STMT, "RestoreMnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_VNODE_STMT, "RestoreVnodeStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_VIEW_STMT, "CreateViewStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_VIEW_STMT, "DropViewStmt") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_SCAN, "LogicScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_JOIN, "LogicJoin") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_AGG, "LogicAgg") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_PROJECT, "LogicProject") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY, "LogicVnodeModify") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_EXCHANGE, "LogicExchange") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_MERGE, "LogicMerge") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_WINDOW, "LogicWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_FILL, "LogicFill") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_SORT, "LogicSort") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_PARTITION, "LogicPartition") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC, "LogicIndefRowsFunc") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_INTERP_FUNC, "LogicInterpFunc") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_GROUP_CACHE, "LogicGroupCache") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL, "LogicDynamicQueryCtrl") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_SUBPLAN, "LogicSubplan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN, "LogicPlan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, "PhysiTagScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, "PhysiTableScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, "PhysiTableSeqScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, "PhysiTableMergeScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, "PhysiSreamScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, "PhysiSystemTableScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, "PhysiBlockDistScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, "PhysiLastRowScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "PhysiTableCountScan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "PhysiMergeEventWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "PhysiStreamEventWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_PROJECT, "PhysiProject") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, "PhysiMergeJoin") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, "PhysiHashJoin") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, "PhysiAgg") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, "PhysiExchange") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE, "PhysiMerge") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_SORT, "PhysiSort") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, "PhysiGroupSort") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, "PhysiHashInterval") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, "PhysiMergeAlignedInterval") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, "PhysiStreamInterval") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, "PhysiStreamFinalInterval") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, "PhysiStreamSemiInterval") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_FILL, "PhysiFill") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, "PhysiFill") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, "PhysiSessionWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, "PhysiStreamSessionWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, "PhysiStreamSemiSessionWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, "PhysiStreamFinalSessionWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, "PhysiStateWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, "PhysiStreamStateWindow") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_PARTITION, "PhysiPartition") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, "PhysiStreamPartition") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, "PhysiIndefRowsFunc") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, "PhysiInterpFunc") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "PhysiDispatch") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_INSERT, "PhysiInsert") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "PhysiQueryInsert") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_DELETE, "PhysiDelete") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, "PhysiGroupCache") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, "PhysiDynamicQueryCtrl") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_SUBPLAN, "PhysiSubplan") + TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN, "PhysiPlan") +} + const char* nodesNodeName(ENodeType type) { - switch (type) { - case QUERY_NODE_COLUMN: - return "Column"; - case QUERY_NODE_VALUE: - return "Value"; - case QUERY_NODE_OPERATOR: - return "Operator"; - case QUERY_NODE_LOGIC_CONDITION: - return "LogicCondition"; - case QUERY_NODE_FUNCTION: - return "Function"; - case QUERY_NODE_REAL_TABLE: - return "RealTable"; - case QUERY_NODE_TEMP_TABLE: - return "TempTable"; - case QUERY_NODE_JOIN_TABLE: - return "JoinTable"; - case QUERY_NODE_GROUPING_SET: - return "GroupingSet"; - case QUERY_NODE_ORDER_BY_EXPR: - return "OrderByExpr"; - case QUERY_NODE_LIMIT: - return "Limit"; - case QUERY_NODE_STATE_WINDOW: - return "StateWindow"; - case QUERY_NODE_SESSION_WINDOW: - return "SessionWinow"; - case QUERY_NODE_INTERVAL_WINDOW: - return "IntervalWindow"; - case QUERY_NODE_NODE_LIST: - return "NodeList"; - case QUERY_NODE_FILL: - return "Fill"; - case QUERY_NODE_RAW_EXPR: - return "RawExpr"; - case QUERY_NODE_TARGET: - return "Target"; - case QUERY_NODE_DATABLOCK_DESC: - return "DataBlockDesc"; - case QUERY_NODE_SLOT_DESC: - return "SlotDesc"; - case QUERY_NODE_COLUMN_DEF: - return "ColumnDef"; - case QUERY_NODE_DOWNSTREAM_SOURCE: - return "DownstreamSource"; - case QUERY_NODE_DATABASE_OPTIONS: - return "DatabaseOptions"; - case QUERY_NODE_TABLE_OPTIONS: - return "TableOptions"; - case QUERY_NODE_INDEX_OPTIONS: - return "IndexOptions"; - case QUERY_NODE_EXPLAIN_OPTIONS: - return "ExplainOptions"; - case QUERY_NODE_STREAM_OPTIONS: - return "StreamOptions"; - case QUERY_NODE_LEFT_VALUE: - return "LeftValue"; - case QUERY_NODE_WHEN_THEN: - return "WhenThen"; - case QUERY_NODE_CASE_WHEN: - return "CaseWhen"; - case QUERY_NODE_EVENT_WINDOW: - return "EventWindow"; - case QUERY_NODE_SET_OPERATOR: - return "SetOperator"; - case QUERY_NODE_SELECT_STMT: - return "SelectStmt"; - case QUERY_NODE_VNODE_MODIFY_STMT: - return "VnodeModifStmt"; - case QUERY_NODE_CREATE_DATABASE_STMT: - return "CreateDatabaseStmt"; - case QUERY_NODE_DROP_DATABASE_STMT: - return "DropDatabaseStmt"; - case QUERY_NODE_ALTER_DATABASE_STMT: - return "AlterDatabaseStmt"; - case QUERY_NODE_FLUSH_DATABASE_STMT: - return "FlushDatabaseStmt"; - case QUERY_NODE_TRIM_DATABASE_STMT: - return "TrimDatabaseStmt"; - case QUERY_NODE_CREATE_TABLE_STMT: - return "CreateTableStmt"; - case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: - return "CreateSubtableClause"; - case QUERY_NODE_CREATE_MULTI_TABLES_STMT: - return "CreateMultiTableStmt"; - case QUERY_NODE_DROP_TABLE_CLAUSE: - return "DropTableClause"; - case QUERY_NODE_DROP_TABLE_STMT: - return "DropTableStmt"; - case QUERY_NODE_DROP_SUPER_TABLE_STMT: - return "DropSuperTableStmt"; - case QUERY_NODE_ALTER_TABLE_STMT: - return "AlterTableStmt"; - case QUERY_NODE_ALTER_SUPER_TABLE_STMT: - return "AlterSuperTableStmt"; - case QUERY_NODE_CREATE_USER_STMT: - return "CreateUserStmt"; - case QUERY_NODE_ALTER_USER_STMT: - return "AlterUserStmt"; - case QUERY_NODE_DROP_USER_STMT: - return "DropUserStmt"; - case QUERY_NODE_USE_DATABASE_STMT: - return "UseDatabaseStmt"; - case QUERY_NODE_CREATE_DNODE_STMT: - return "CreateDnodeStmt"; - case QUERY_NODE_DROP_DNODE_STMT: - return "DropDnodeStmt"; - case QUERY_NODE_ALTER_DNODE_STMT: - return "AlterDnodeStmt"; - case QUERY_NODE_CREATE_INDEX_STMT: - return "CreateIndexStmt"; - case QUERY_NODE_DROP_INDEX_STMT: - return "DropIndexStmt"; - case QUERY_NODE_CREATE_QNODE_STMT: - return "CreateQnodeStmt"; - case QUERY_NODE_DROP_QNODE_STMT: - return "DropQnodeStmt"; - case QUERY_NODE_CREATE_SNODE_STMT: - return "CreateSnodeStmt"; - case QUERY_NODE_DROP_SNODE_STMT: - return "DropSnodeStmt"; - case QUERY_NODE_CREATE_MNODE_STMT: - return "CreateMnodeStmt"; - case QUERY_NODE_DROP_MNODE_STMT: - return "DropMnodeStmt"; - case QUERY_NODE_CREATE_TOPIC_STMT: - return "CreateTopicStmt"; - case QUERY_NODE_DROP_TOPIC_STMT: - return "DropTopicStmt"; - case QUERY_NODE_DROP_CGROUP_STMT: - return "DropConsumerGroupStmt"; - case QUERY_NODE_ALTER_LOCAL_STMT: - return "AlterLocalStmt"; - case QUERY_NODE_EXPLAIN_STMT: - return "ExplainStmt"; - case QUERY_NODE_DESCRIBE_STMT: - return "DescribeStmt"; - case QUERY_NODE_COMPACT_DATABASE_STMT: - return "CompactDatabaseStmt"; - case QUERY_NODE_CREATE_STREAM_STMT: - return "CreateStreamStmt"; - case QUERY_NODE_DROP_STREAM_STMT: - return "DropStreamStmt"; - case QUERY_NODE_PAUSE_STREAM_STMT: - return "PauseStreamStmt"; - case QUERY_NODE_RESUME_STREAM_STMT: - return "ResumeStreamStmt"; - case QUERY_NODE_BALANCE_VGROUP_STMT: - return "BalanceVgroupStmt"; - case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: - return "BalanceVgroupLeaderStmt"; - case QUERY_NODE_MERGE_VGROUP_STMT: - return "MergeVgroupStmt"; - case QUERY_NODE_SHOW_DB_ALIVE_STMT: - return "ShowDbAliveStmt"; - case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: - return "ShowClusterAliveStmt"; - case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: - return "RedistributeVgroupStmt"; - case QUERY_NODE_SPLIT_VGROUP_STMT: - return "SplitVgroupStmt"; - case QUERY_NODE_GRANT_STMT: - return "GrantStmt"; - case QUERY_NODE_REVOKE_STMT: - return "RevokeStmt"; - case QUERY_NODE_SHOW_DNODES_STMT: - return "ShowDnodesStmt"; - case QUERY_NODE_SHOW_MNODES_STMT: - return "ShowMnodesStmt"; - case QUERY_NODE_SHOW_MODULES_STMT: - return "ShowModulesStmt"; - case QUERY_NODE_SHOW_QNODES_STMT: - return "ShowQnodesStmt"; - case QUERY_NODE_SHOW_SNODES_STMT: - return "ShowSnodesStmt"; - case QUERY_NODE_SHOW_BNODES_STMT: - return "ShowBnodesStmt"; - case QUERY_NODE_SHOW_CLUSTER_STMT: - return "ShowClusterStmt"; - case QUERY_NODE_SHOW_DATABASES_STMT: - return "ShowDatabaseStmt"; - case QUERY_NODE_SHOW_FUNCTIONS_STMT: - return "ShowFunctionsStmt"; - case QUERY_NODE_SHOW_INDEXES_STMT: - return "ShowIndexesStmt"; - case QUERY_NODE_SHOW_STABLES_STMT: - return "ShowStablesStmt"; - case QUERY_NODE_SHOW_STREAMS_STMT: - return "ShowStreamsStmt"; - case QUERY_NODE_SHOW_TABLES_STMT: - return "ShowTablesStmt"; - case QUERY_NODE_SHOW_TAGS_STMT: - return "ShowTagsStmt"; - case QUERY_NODE_SHOW_USERS_STMT: - return "ShowUsersStmt"; - case QUERY_NODE_SHOW_LICENCES_STMT: - return "ShowGrantsStmt"; - case QUERY_NODE_SHOW_VGROUPS_STMT: - return "ShowVgroupsStmt"; - case QUERY_NODE_SHOW_TOPICS_STMT: - return "ShowTopicsStmt"; - case QUERY_NODE_SHOW_CONSUMERS_STMT: - return "ShowConsumersStmt"; - case QUERY_NODE_SHOW_QUERIES_STMT: - return "ShowQueriesStmt"; - case QUERY_NODE_SHOW_VARIABLES_STMT: - return "ShowVariablesStmt"; - case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: - return "ShowDnodeVariablesStmt"; - case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - return "ShowTransactionsStmt"; - case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: - return "ShowSubscriptionsStmt"; - case QUERY_NODE_SHOW_VNODES_STMT: - return "ShowVnodeStmt"; - case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: - return "ShowUserPrivilegesStmt"; - case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: - return "ShowCreateDatabasesStmt"; - case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - return "ShowCreateTablesStmt"; - case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - return "ShowCreateStablesStmt"; - case QUERY_NODE_SHOW_CREATE_VIEW_STMT: - return "ShowCreateViewStmt"; - case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: - return "ShowTableDistributedStmt"; - case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: - return "ShowLocalVariablesStmt"; - case QUERY_NODE_SHOW_TABLE_TAGS_STMT: - return "ShowTableTagsStmt"; - case QUERY_NODE_DELETE_STMT: - return "DeleteStmt"; - case QUERY_NODE_INSERT_STMT: - return "InsertStmt"; - case QUERY_NODE_RESTORE_DNODE_STMT: - return "RestoreDnodeStmt"; - case QUERY_NODE_RESTORE_QNODE_STMT: - return "RestoreQnodeStmt"; - case QUERY_NODE_RESTORE_MNODE_STMT: - return "RestoreMnodeStmt"; - case QUERY_NODE_RESTORE_VNODE_STMT: - return "RestoreVnodeStmt"; - case QUERY_NODE_CREATE_VIEW_STMT: - return "CreateViewStmt"; - case QUERY_NODE_DROP_VIEW_STMT: - return "DropViewStmt"; - case QUERY_NODE_LOGIC_PLAN_SCAN: - return "LogicScan"; - case QUERY_NODE_LOGIC_PLAN_JOIN: - return "LogicJoin"; - case QUERY_NODE_LOGIC_PLAN_AGG: - return "LogicAgg"; - case QUERY_NODE_LOGIC_PLAN_PROJECT: - return "LogicProject"; - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: - return "LogicVnodeModify"; - case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - return "LogicExchange"; - case QUERY_NODE_LOGIC_PLAN_MERGE: - return "LogicMerge"; - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return "LogicWindow"; - case QUERY_NODE_LOGIC_PLAN_FILL: - return "LogicFill"; - case QUERY_NODE_LOGIC_PLAN_SORT: - return "LogicSort"; - case QUERY_NODE_LOGIC_PLAN_PARTITION: - return "LogicPartition"; - case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: - return "LogicIndefRowsFunc"; - case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: - return "LogicInterpFunc"; - case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: - return "LogicGroupCache"; - case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL: - return "LogicDynamicQueryCtrl"; - case QUERY_NODE_LOGIC_SUBPLAN: - return "LogicSubplan"; - case QUERY_NODE_LOGIC_PLAN: - return "LogicPlan"; - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - return "PhysiTagScan"; - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - return "PhysiTableScan"; - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - return "PhysiTableSeqScan"; - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - return "PhysiTableMergeScan"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return "PhysiSreamScan"; - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - return "PhysiSystemTableScan"; - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - return "PhysiBlockDistScan"; - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - return "PhysiLastRowScan"; - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: - return "PhysiTableCountScan"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: - return "PhysiMergeEventWindow"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: - return "PhysiStreamEventWindow"; - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - return "PhysiProject"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return "PhysiMergeJoin"; - case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: - return "PhysiHashJoin"; - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: - return "PhysiAgg"; - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - return "PhysiExchange"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE: - return "PhysiMerge"; - case QUERY_NODE_PHYSICAL_PLAN_SORT: - return "PhysiSort"; - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: - return "PhysiGroupSort"; - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - return "PhysiHashInterval"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: - return "PhysiMergeAlignedInterval"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - return "PhysiStreamInterval"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: - return "PhysiStreamFinalInterval"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - return "PhysiStreamSemiInterval"; - case QUERY_NODE_PHYSICAL_PLAN_FILL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: - return "PhysiFill"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: - return "PhysiSessionWindow"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: - return "PhysiStreamSessionWindow"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: - return "PhysiStreamSemiSessionWindow"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - return "PhysiStreamFinalSessionWindow"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: - return "PhysiStateWindow"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: - return "PhysiStreamStateWindow"; - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: - return "PhysiPartition"; - case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: - return "PhysiStreamPartition"; - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: - return "PhysiIndefRowsFunc"; - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: - return "PhysiInterpFunc"; - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - return "PhysiDispatch"; - case QUERY_NODE_PHYSICAL_PLAN_INSERT: - return "PhysiInsert"; - case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: - return "PhysiQueryInsert"; - case QUERY_NODE_PHYSICAL_PLAN_DELETE: - return "PhysiDelete"; - case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: - return "PhysiGroupCache"; - case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: - return "PhysiDynamicQueryCtrl"; - case QUERY_NODE_PHYSICAL_SUBPLAN: - return "PhysiSubplan"; - case QUERY_NODE_PHYSICAL_PLAN: - return "PhysiPlan"; - default: - break; - } + if (nodeNames[type]) + return nodeNames[type]; + nodesWarn("nodesNodeName unknown node = %d", type); return "UnknownNode"; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d167d81c82..c8de8bdfac 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -131,6 +131,7 @@ static void destroyNodeAllocator(void* p) { } int32_t nodesInitAllocatorSet() { + initNodeName(); if (g_allocatorReqRefPool >= 0) { nodesWarn("nodes already initialized"); return TSDB_CODE_SUCCESS; From e453962271946fc458cad84b0e00b27cdcb91890 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 14 Nov 2023 15:21:26 +0800 Subject: [PATCH 02/43] rewrite nodes function --- include/libs/nodes/nodes.h | 3 +- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 2 + source/libs/nodes/src/nodesCodeFuncs.c | 3092 +++++++++++++++++------ source/libs/nodes/src/nodesUtilFuncs.c | 1241 +-------- 4 files changed, 2262 insertions(+), 2076 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 21d33249b8..136926a856 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -101,7 +101,7 @@ typedef struct SNodeList { typedef struct SNodeAllocator SNodeAllocator; -void initNodeName(); +void nodesInit(); int32_t nodesInitAllocatorSet(); void nodesDestroyAllocatorSet(); int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId); @@ -111,6 +111,7 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId); int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId); void nodesDestroyAllocator(int64_t allocatorId); +int32_t getNodeSize(ENodeType type); SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); void nodesFree(void* p); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 6f13abcebc..c57043b8bb 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -17,6 +17,7 @@ #include "dmMgmt.h" #include "audit.h" #include "libs/function/tudf.h" +#include "nodes.h" #define DM_INIT_AUDIT() \ do { \ @@ -155,6 +156,7 @@ extern void s3End(); int32_t dmInit() { dInfo("start to init dnode env"); + nodesInit(); if (dmDiskInit() != 0) return -1; if (!dmCheckDataDirVersion()) return -1; if (!dmCheckDiskSpace()) return -1; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 582ecf6753..01d808c7d8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -28,204 +28,67 @@ static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** p static int32_t makeNodeByJson(const SJson* pJson, SNode** pNode); -typedef const char* taoscstr; -taoscstr nodeNames[QUERY_NODE_END] = {}; +typedef int32_t (*FExecNodeToJson)(const void* pObj, SJson* pJson); +typedef int32_t (*FExecJsonToNode)(const SJson* pJson, void* pObj); +typedef void (*FExecDestoryNode)(SNode* pNode); -#define TAOS_DEFINE_NODE_NAME(type, msg) nodeNames[type] = msg; +/** + * @brief Node operation to binding function set + */ +typedef struct SBuiltinNodeDefinition { + const char* name; + int32_t nodeSize; + FExecNodeToJson toJsonFunc; + FExecJsonToNode toNodeFunc; + FExecDestoryNode destoryFunc; +} SBuiltinNodeDefinition; -void initNodeName() { - TAOS_DEFINE_NODE_NAME(QUERY_NODE_COLUMN, "Column") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_VALUE, "Value") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_OPERATOR, "Operator") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_CONDITION, "LogicCondition") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_FUNCTION, "Function") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_REAL_TABLE, "RealTable") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_TEMP_TABLE, "TempTable") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_JOIN_TABLE, "JoinTable") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_GROUPING_SET, "GroupingSet") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ORDER_BY_EXPR, "OrderByExpr") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LIMIT, "Limit") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_STATE_WINDOW, "StateWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SESSION_WINDOW, "SessionWinow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_INTERVAL_WINDOW, "IntervalWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_NODE_LIST, "NodeList") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_FILL, "Fill") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_RAW_EXPR, "RawExpr") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_TARGET, "Target") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DATABLOCK_DESC, "DataBlockDesc") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SLOT_DESC, "SlotDesc") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_COLUMN_DEF, "ColumnDef") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DOWNSTREAM_SOURCE, "DownstreamSource") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DATABASE_OPTIONS, "DatabaseOptions") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_TABLE_OPTIONS, "TableOptions") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_INDEX_OPTIONS, "IndexOptions") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_EXPLAIN_OPTIONS, "ExplainOptions") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_STREAM_OPTIONS, "StreamOptions") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LEFT_VALUE, "LeftValue") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_WHEN_THEN, "WhenThen") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CASE_WHEN, "CaseWhen") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_EVENT_WINDOW, "EventWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SET_OPERATOR, "SetOperator") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SELECT_STMT, "SelectStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_VNODE_MODIFY_STMT, "VnodeModifStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_DATABASE_STMT, "CreateDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_DATABASE_STMT, "DropDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_DATABASE_STMT, "AlterDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_FLUSH_DATABASE_STMT, "FlushDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_TRIM_DATABASE_STMT, "TrimDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_TABLE_STMT, "CreateTableStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_SUBTABLE_CLAUSE, "CreateSubtableClause") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_MULTI_TABLES_STMT, "CreateMultiTableStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_TABLE_CLAUSE, "DropTableClause") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_TABLE_STMT, "DropTableStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_SUPER_TABLE_STMT, "DropSuperTableStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_TABLE_STMT, "AlterTableStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_SUPER_TABLE_STMT, "AlterSuperTableStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_USER_STMT, "CreateUserStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_USER_STMT, "AlterUserStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_USER_STMT, "DropUserStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_USE_DATABASE_STMT, "UseDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_DNODE_STMT, "CreateDnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_DNODE_STMT, "DropDnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_DNODE_STMT, "AlterDnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_INDEX_STMT, "CreateIndexStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_INDEX_STMT, "DropIndexStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_QNODE_STMT, "CreateQnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_QNODE_STMT, "DropQnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_SNODE_STMT, "CreateSnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_SNODE_STMT, "DropSnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_MNODE_STMT, "CreateMnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_MNODE_STMT, "DropMnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_TOPIC_STMT, "CreateTopicStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_TOPIC_STMT, "DropTopicStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_CGROUP_STMT, "DropConsumerGroupStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_ALTER_LOCAL_STMT, "AlterLocalStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_EXPLAIN_STMT, "ExplainStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DESCRIBE_STMT, "DescribeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_COMPACT_DATABASE_STMT, "CompactDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_STREAM_STMT, "CreateStreamStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_STREAM_STMT, "DropStreamStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PAUSE_STREAM_STMT, "PauseStreamStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESUME_STREAM_STMT, "ResumeStreamStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_BALANCE_VGROUP_STMT, "BalanceVgroupStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, "BalanceVgroupLeaderStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_MERGE_VGROUP_STMT, "MergeVgroupStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DB_ALIVE_STMT, "ShowDbAliveStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, "ShowClusterAliveStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT, "RedistributeVgroupStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SPLIT_VGROUP_STMT, "SplitVgroupStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_GRANT_STMT, "GrantStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_REVOKE_STMT, "RevokeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DNODES_STMT, "ShowDnodesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_MNODES_STMT, "ShowMnodesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_MODULES_STMT, "ShowModulesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_QNODES_STMT, "ShowQnodesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_SNODES_STMT, "ShowSnodesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_BNODES_STMT, "ShowBnodesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CLUSTER_STMT, "ShowClusterStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DATABASES_STMT, "ShowDatabaseStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_FUNCTIONS_STMT, "ShowFunctionsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_INDEXES_STMT, "ShowIndexesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_STABLES_STMT, "ShowStablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_STREAMS_STMT, "ShowStreamsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TABLES_STMT, "ShowTablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TAGS_STMT, "ShowTagsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_USERS_STMT, "ShowUsersStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_LICENCES_STMT, "ShowGrantsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_VGROUPS_STMT, "ShowVgroupsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TOPICS_STMT, "ShowTopicsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CONSUMERS_STMT, "ShowConsumersStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_QUERIES_STMT, "ShowQueriesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_VARIABLES_STMT, "ShowVariablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_DNODE_VARIABLES_STMT, "ShowDnodeVariablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TRANSACTIONS_STMT, "ShowTransactionsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, "ShowSubscriptionsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_VNODES_STMT, "ShowVnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, "ShowUserPrivilegesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_DATABASE_STMT, "ShowCreateDatabasesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_TABLE_STMT, "ShowCreateTablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_STABLE_STMT, "ShowCreateStablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_CREATE_VIEW_STMT, "ShowCreateViewStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, "ShowTableDistributedStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, "ShowLocalVariablesStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_SHOW_TABLE_TAGS_STMT, "ShowTableTagsStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DELETE_STMT, "DeleteStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_INSERT_STMT, "InsertStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_DNODE_STMT, "RestoreDnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_QNODE_STMT, "RestoreQnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_MNODE_STMT, "RestoreMnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_RESTORE_VNODE_STMT, "RestoreVnodeStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_CREATE_VIEW_STMT, "CreateViewStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_DROP_VIEW_STMT, "DropViewStmt") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_SCAN, "LogicScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_JOIN, "LogicJoin") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_AGG, "LogicAgg") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_PROJECT, "LogicProject") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY, "LogicVnodeModify") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_EXCHANGE, "LogicExchange") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_MERGE, "LogicMerge") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_WINDOW, "LogicWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_FILL, "LogicFill") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_SORT, "LogicSort") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_PARTITION, "LogicPartition") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC, "LogicIndefRowsFunc") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_INTERP_FUNC, "LogicInterpFunc") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_GROUP_CACHE, "LogicGroupCache") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL, "LogicDynamicQueryCtrl") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_SUBPLAN, "LogicSubplan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_LOGIC_PLAN, "LogicPlan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, "PhysiTagScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, "PhysiTableScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, "PhysiTableSeqScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, "PhysiTableMergeScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, "PhysiSreamScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, "PhysiSystemTableScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, "PhysiBlockDistScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, "PhysiLastRowScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "PhysiTableCountScan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "PhysiMergeEventWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "PhysiStreamEventWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_PROJECT, "PhysiProject") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, "PhysiMergeJoin") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, "PhysiHashJoin") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, "PhysiAgg") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, "PhysiExchange") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE, "PhysiMerge") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_SORT, "PhysiSort") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, "PhysiGroupSort") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, "PhysiHashInterval") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, "PhysiMergeAlignedInterval") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, "PhysiStreamInterval") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, "PhysiStreamFinalInterval") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, "PhysiStreamSemiInterval") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_FILL, "PhysiFill") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, "PhysiFill") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, "PhysiSessionWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, "PhysiStreamSessionWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, "PhysiStreamSemiSessionWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, "PhysiStreamFinalSessionWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, "PhysiStateWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, "PhysiStreamStateWindow") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_PARTITION, "PhysiPartition") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, "PhysiStreamPartition") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, "PhysiIndefRowsFunc") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, "PhysiInterpFunc") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "PhysiDispatch") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_INSERT, "PhysiInsert") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "PhysiQueryInsert") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_DELETE, "PhysiDelete") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, "PhysiGroupCache") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, "PhysiDynamicQueryCtrl") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_SUBPLAN, "PhysiSubplan") - TAOS_DEFINE_NODE_NAME(QUERY_NODE_PHYSICAL_PLAN, "PhysiPlan") +SBuiltinNodeDefinition funcNodes[QUERY_NODE_END] = {}; + +static TdThreadOnce functionNodeInit = PTHREAD_ONCE_INIT; +static int32_t initNodeCode = -1; + +static void setFunc(const char* name, int32_t type, int32_t nodeSize, FExecNodeToJson toJsonFunc, + FExecJsonToNode toNodeFunc, FExecDestoryNode destoryFunc) { + funcNodes[type].name = name; + funcNodes[type].nodeSize = nodeSize; + funcNodes[type].toJsonFunc = toJsonFunc; + funcNodes[type].toNodeFunc = toNodeFunc; + funcNodes[type].destoryFunc = destoryFunc; +} + +static void doInitNodeFuncArray(); + +void nodesInit() { + taosThreadOnce(&functionNodeInit, doInitNodeFuncArray); +} + +bool funcArrayCheck(int32_t type) { + if (type < 0 || QUERY_NODE_END < (type+1)) { + nodesError("funcArrayCheck unknown type = %d", type); + return false; + } + if (initNodeCode != 0) { + nodesInit(); + } + if (!funcNodes[type].name) { + return false; + } + return true; +} + +int32_t getNodeSize(ENodeType type) { + if (!funcArrayCheck(type)) { + return 0; + } + return funcNodes[type].nodeSize; } const char* nodesNodeName(ENodeType type) { - if (nodeNames[type]) - return nodeNames[type]; - - nodesWarn("nodesNodeName unknown node = %d", type); - return "UnknownNode"; + if (!funcArrayCheck(type)) { + return NULL; + } + return funcNodes[type].name; } static int32_t nodeListToJson(SJson* pJson, const char* pName, const SNodeList* pList) { @@ -6628,657 +6491,31 @@ static int32_t jsonToInsertStmt(const SJson* pJson, void* pObj) { return code; } -static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { - switch (nodeType(pObj)) { - case QUERY_NODE_COLUMN: - return columnNodeToJson(pObj, pJson); - case QUERY_NODE_VALUE: - return valueNodeToJson(pObj, pJson); - case QUERY_NODE_OPERATOR: - return operatorNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_CONDITION: - return logicConditionNodeToJson(pObj, pJson); - case QUERY_NODE_FUNCTION: - return functionNodeToJson(pObj, pJson); - case QUERY_NODE_REAL_TABLE: - return realTableNodeToJson(pObj, pJson); - case QUERY_NODE_TEMP_TABLE: - return tempTableNodeToJson(pObj, pJson); - case QUERY_NODE_JOIN_TABLE: - return joinTableNodeToJson(pObj, pJson); - case QUERY_NODE_GROUPING_SET: - return groupingSetNodeToJson(pObj, pJson); - case QUERY_NODE_ORDER_BY_EXPR: - return orderByExprNodeToJson(pObj, pJson); - case QUERY_NODE_LIMIT: - return limitNodeToJson(pObj, pJson); - case QUERY_NODE_STATE_WINDOW: - return stateWindowNodeToJson(pObj, pJson); - case QUERY_NODE_SESSION_WINDOW: - return sessionWindowNodeToJson(pObj, pJson); - case QUERY_NODE_INTERVAL_WINDOW: - return intervalWindowNodeToJson(pObj, pJson); - case QUERY_NODE_NODE_LIST: - return nodeListNodeToJson(pObj, pJson); - case QUERY_NODE_FILL: - return fillNodeToJson(pObj, pJson); - case QUERY_NODE_RAW_EXPR: - break; - case QUERY_NODE_TARGET: - return targetNodeToJson(pObj, pJson); - case QUERY_NODE_DATABLOCK_DESC: - return dataBlockDescNodeToJson(pObj, pJson); - case QUERY_NODE_SLOT_DESC: - return slotDescNodeToJson(pObj, pJson); - case QUERY_NODE_COLUMN_DEF: - return columnDefNodeToJson(pObj, pJson); - case QUERY_NODE_DOWNSTREAM_SOURCE: - return downstreamSourceNodeToJson(pObj, pJson); - case QUERY_NODE_DATABASE_OPTIONS: - return databaseOptionsToJson(pObj, pJson); - case QUERY_NODE_TABLE_OPTIONS: - return tableOptionsToJson(pObj, pJson); - case QUERY_NODE_INDEX_OPTIONS: - return indexOptionsToJson(pObj, pJson); - case QUERY_NODE_EXPLAIN_OPTIONS: - return explainOptionsToJson(pObj, pJson); - case QUERY_NODE_STREAM_OPTIONS: - return streamOptionsToJson(pObj, pJson); - case QUERY_NODE_LEFT_VALUE: - return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to serialize. - case QUERY_NODE_WHEN_THEN: - return whenThenNodeToJson(pObj, pJson); - case QUERY_NODE_CASE_WHEN: - return caseWhenNodeToJson(pObj, pJson); - case QUERY_NODE_EVENT_WINDOW: - return eventWindowNodeToJson(pObj, pJson); - case QUERY_NODE_SET_OPERATOR: - return setOperatorToJson(pObj, pJson); - case QUERY_NODE_SELECT_STMT: - return selectStmtToJson(pObj, pJson); - case QUERY_NODE_VNODE_MODIFY_STMT: - return vnodeModifyStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_DATABASE_STMT: - return createDatabaseStmtToJson(pObj, pJson); - case QUERY_NODE_ALTER_DATABASE_STMT: - return alterDatabaseStmtToJson(pObj, pJson); - case QUERY_NODE_TRIM_DATABASE_STMT: - return trimDatabaseStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_TABLE_STMT: - return createTableStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: - return createSubTableClauseToJson(pObj, pJson); - case QUERY_NODE_CREATE_MULTI_TABLES_STMT: - return createMultiTablesStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_TABLE_CLAUSE: - return dropTableClauseToJson(pObj, pJson); - case QUERY_NODE_DROP_TABLE_STMT: - return dropTableStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_SUPER_TABLE_STMT: - return dropStableStmtToJson(pObj, pJson); - case QUERY_NODE_ALTER_TABLE_STMT: - return alterTableStmtToJson(pObj, pJson); - case QUERY_NODE_ALTER_SUPER_TABLE_STMT: - return alterStableStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_USER_STMT: - return createUserStmtToJson(pObj, pJson); - case QUERY_NODE_ALTER_USER_STMT: - return alterUserStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_USER_STMT: - return dropUserStmtToJson(pObj, pJson); - case QUERY_NODE_USE_DATABASE_STMT: - return useDatabaseStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_DNODE_STMT: - return createDnodeStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_DNODE_STMT: - return dropDnodeStmtToJson(pObj, pJson); - case QUERY_NODE_ALTER_DNODE_STMT: - return alterDnodeStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_INDEX_STMT: - return createIndexStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_INDEX_STMT: - return dropIndexStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_QNODE_STMT: - return createQnodeStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_QNODE_STMT: - return dropQnodeStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_SNODE_STMT: - return createSnodeStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_SNODE_STMT: - return dropSnodeStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_MNODE_STMT: - return createMnodeStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_MNODE_STMT: - return dropMnodeStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_TOPIC_STMT: - return createTopicStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_TOPIC_STMT: - return dropTopicStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_CGROUP_STMT: - return dropConsumerGroupStmtToJson(pObj, pJson); - case QUERY_NODE_ALTER_LOCAL_STMT: - return alterLocalStmtToJson(pObj, pJson); - case QUERY_NODE_EXPLAIN_STMT: - return explainStmtToJson(pObj, pJson); - case QUERY_NODE_DESCRIBE_STMT: - return describeStmtToJson(pObj, pJson); - case QUERY_NODE_COMPACT_DATABASE_STMT: - return compactDatabaseStmtToJson(pObj, pJson); - case QUERY_NODE_CREATE_STREAM_STMT: - return createStreamStmtToJson(pObj, pJson); - case QUERY_NODE_DROP_STREAM_STMT: - return dropStreamStmtToJson(pObj, pJson); - case QUERY_NODE_BALANCE_VGROUP_STMT: - return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize. - case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: - return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to serialize. - case QUERY_NODE_MERGE_VGROUP_STMT: - return mergeVgroupStmtToJson(pObj, pJson); - case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: - return redistributeVgroupStmtToJson(pObj, pJson); - case QUERY_NODE_SPLIT_VGROUP_STMT: - return splitVgroupStmtToJson(pObj, pJson); - case QUERY_NODE_GRANT_STMT: - return grantStmtToJson(pObj, pJson); - case QUERY_NODE_REVOKE_STMT: - return revokeStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_DNODES_STMT: - return showDnodesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_MNODES_STMT: - return showMnodesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_QNODES_STMT: - return showQnodesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_CLUSTER_STMT: - return showClusterStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_DATABASES_STMT: - return showDatabasesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_FUNCTIONS_STMT: - return showFunctionsStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_INDEXES_STMT: - return showIndexesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_STABLES_STMT: - return showStablesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_STREAMS_STMT: - return showStreamsStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_TABLES_STMT: - return showTablesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_TAGS_STMT: - return showTagsStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_USERS_STMT: - return showUsersStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_VGROUPS_STMT: - return showVgroupsStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_CONSUMERS_STMT: - return showConsumersStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_VARIABLES_STMT: - return showVariablesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: - return showDnodeVariablesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - return showTransactionsStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: - return showSubscriptionsStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_VNODES_STMT: - return showVnodesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: - return showUserPrivilegesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: - return showCreateDatabaseStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - return showCreateTableStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - return showCreateStableStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_CREATE_VIEW_STMT: - return showCreateViewStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: - return showTableDistributedStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: - return showLocalVariablesStmtToJson(pObj, pJson); - case QUERY_NODE_SHOW_TABLE_TAGS_STMT: - return showTableTagsStmtToJson(pObj, pJson); - case QUERY_NODE_DELETE_STMT: - return deleteStmtToJson(pObj, pJson); - case QUERY_NODE_INSERT_STMT: - return insertStmtToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_SCAN: - return logicScanNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_JOIN: - return logicJoinNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_AGG: - return logicAggNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_PROJECT: - return logicProjectNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: - return logicVnodeModifyNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - return logicExchangeNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_MERGE: - return logicMergeNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return logicWindowNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_FILL: - return logicFillNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_SORT: - return logicSortNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_PARTITION: - return logicPartitionNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: - return logicIndefRowsFuncNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: - return logicInterpFuncNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: - return logicGroupCacheNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL: - return logicDynQueryCtrlNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_SUBPLAN: - return logicSubplanToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN: - return logicPlanToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - return physiTagScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - return physiScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: - return physiLastRowScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return physiTableScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - return physiSysTableScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - return physiProjectNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return physiMergeJoinNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: - return physiHashJoinNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: - return physiAggNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - return physiExchangeNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_MERGE: - return physiMergeNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_SORT: - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: - return physiSortNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - return physiIntervalNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_FILL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: - return physiFillNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - return physiSessionWindowNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: - return physiStateWindowNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: - return physiEventWindowNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: - return physiPartitionNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: - return physiStreamPartitionNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: - return physiIndefRowsFuncNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: - return physiInterpFuncNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - return physiDispatchNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_INSERT: - break; - case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: - return physiQueryInsertNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_DELETE: - return physiDeleteNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: - return physiGroupCacheNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: - return physiDynQueryCtrlNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_SUBPLAN: - return subplanToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN: - return planToJson(pObj, pJson); - default: - break; +int32_t specificNodeToJson(const void* pObj, SJson* pJson) { + ENodeType type = nodeType(pObj); + if (!funcArrayCheck(type)) { + return TSDB_CODE_SUCCESS; } - nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); + + if (funcNodes[type].toJsonFunc) { + return funcNodes[type].toJsonFunc(pObj, pJson); + } + + nodesWarn("specificNodeToJson unknown node type = %d", type); return TSDB_CODE_SUCCESS; } -static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { - switch (nodeType(pObj)) { - case QUERY_NODE_COLUMN: - return jsonToColumnNode(pJson, pObj); - case QUERY_NODE_VALUE: - return jsonToValueNode(pJson, pObj); - case QUERY_NODE_OPERATOR: - return jsonToOperatorNode(pJson, pObj); - case QUERY_NODE_LOGIC_CONDITION: - return jsonToLogicConditionNode(pJson, pObj); - case QUERY_NODE_FUNCTION: - return jsonToFunctionNode(pJson, pObj); - case QUERY_NODE_REAL_TABLE: - return jsonToRealTableNode(pJson, pObj); - case QUERY_NODE_TEMP_TABLE: - return jsonToTempTableNode(pJson, pObj); - case QUERY_NODE_JOIN_TABLE: - return jsonToJoinTableNode(pJson, pObj); - case QUERY_NODE_GROUPING_SET: - return jsonToGroupingSetNode(pJson, pObj); - case QUERY_NODE_ORDER_BY_EXPR: - return jsonToOrderByExprNode(pJson, pObj); - case QUERY_NODE_LIMIT: - return jsonToLimitNode(pJson, pObj); - case QUERY_NODE_STATE_WINDOW: - return jsonToStateWindowNode(pJson, pObj); - case QUERY_NODE_SESSION_WINDOW: - return jsonToSessionWindowNode(pJson, pObj); - case QUERY_NODE_INTERVAL_WINDOW: - return jsonToIntervalWindowNode(pJson, pObj); - case QUERY_NODE_NODE_LIST: - return jsonToNodeListNode(pJson, pObj); - case QUERY_NODE_FILL: - return jsonToFillNode(pJson, pObj); - case QUERY_NODE_TARGET: - return jsonToTargetNode(pJson, pObj); - case QUERY_NODE_DATABLOCK_DESC: - return jsonToDataBlockDescNode(pJson, pObj); - case QUERY_NODE_SLOT_DESC: - return jsonToSlotDescNode(pJson, pObj); - case QUERY_NODE_COLUMN_DEF: - return jsonToColumnDefNode(pJson, pObj); - case QUERY_NODE_DOWNSTREAM_SOURCE: - return jsonToDownstreamSourceNode(pJson, pObj); - case QUERY_NODE_DATABASE_OPTIONS: - return jsonToDatabaseOptions(pJson, pObj); - case QUERY_NODE_TABLE_OPTIONS: - return jsonToTableOptions(pJson, pObj); - case QUERY_NODE_INDEX_OPTIONS: - return jsonToIndexOptions(pJson, pObj); - case QUERY_NODE_EXPLAIN_OPTIONS: - return jsonToExplainOptions(pJson, pObj); - case QUERY_NODE_STREAM_OPTIONS: - return jsonToStreamOptions(pJson, pObj); - case QUERY_NODE_LEFT_VALUE: - return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to deserialize. - case QUERY_NODE_WHEN_THEN: - return jsonToWhenThenNode(pJson, pObj); - case QUERY_NODE_CASE_WHEN: - return jsonToCaseWhenNode(pJson, pObj); - case QUERY_NODE_EVENT_WINDOW: - return jsonToEventWindowNode(pJson, pObj); - case QUERY_NODE_SET_OPERATOR: - return jsonToSetOperator(pJson, pObj); - case QUERY_NODE_SELECT_STMT: - return jsonToSelectStmt(pJson, pObj); - case QUERY_NODE_VNODE_MODIFY_STMT: - return jsonToVnodeModifyStmt(pJson, pObj); - case QUERY_NODE_CREATE_DATABASE_STMT: - return jsonToCreateDatabaseStmt(pJson, pObj); - case QUERY_NODE_ALTER_DATABASE_STMT: - return jsonToAlterDatabaseStmt(pJson, pObj); - case QUERY_NODE_TRIM_DATABASE_STMT: - return jsonToTrimDatabaseStmt(pJson, pObj); - case QUERY_NODE_CREATE_TABLE_STMT: - return jsonToCreateTableStmt(pJson, pObj); - case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: - return jsonToCreateSubTableClause(pJson, pObj); - case QUERY_NODE_CREATE_MULTI_TABLES_STMT: - return jsonToCreateMultiTablesStmt(pJson, pObj); - case QUERY_NODE_DROP_TABLE_CLAUSE: - return jsonToDropTableClause(pJson, pObj); - case QUERY_NODE_DROP_TABLE_STMT: - return jsonToDropTableStmt(pJson, pObj); - case QUERY_NODE_DROP_SUPER_TABLE_STMT: - return jsonToDropStableStmt(pJson, pObj); - case QUERY_NODE_ALTER_TABLE_STMT: - return jsonToAlterTableStmt(pJson, pObj); - case QUERY_NODE_ALTER_SUPER_TABLE_STMT: - return jsonToAlterStableStmt(pJson, pObj); - case QUERY_NODE_CREATE_USER_STMT: - return jsonToCreateUserStmt(pJson, pObj); - case QUERY_NODE_ALTER_USER_STMT: - return jsonToAlterUserStmt(pJson, pObj); - case QUERY_NODE_DROP_USER_STMT: - return jsonToDropUserStmt(pJson, pObj); - case QUERY_NODE_USE_DATABASE_STMT: - return jsonToUseDatabaseStmt(pJson, pObj); - case QUERY_NODE_CREATE_DNODE_STMT: - return jsonToCreateDnodeStmt(pJson, pObj); - case QUERY_NODE_DROP_DNODE_STMT: - return jsonToDropDnodeStmt(pJson, pObj); - case QUERY_NODE_ALTER_DNODE_STMT: - return jsonToAlterDnodeStmt(pJson, pObj); - case QUERY_NODE_CREATE_INDEX_STMT: - return jsonToCreateIndexStmt(pJson, pObj); - case QUERY_NODE_DROP_INDEX_STMT: - return jsonToDropIndexStmt(pJson, pObj); - case QUERY_NODE_CREATE_QNODE_STMT: - return jsonToCreateQnodeStmt(pJson, pObj); - case QUERY_NODE_DROP_QNODE_STMT: - return jsonToDropQnodeStmt(pJson, pObj); - case QUERY_NODE_CREATE_SNODE_STMT: - return jsonToCreateSnodeStmt(pJson, pObj); - case QUERY_NODE_DROP_SNODE_STMT: - return jsonToDropSnodeStmt(pJson, pObj); - case QUERY_NODE_CREATE_MNODE_STMT: - return jsonToCreateMnodeStmt(pJson, pObj); - case QUERY_NODE_DROP_MNODE_STMT: - return jsonToDropMnodeStmt(pJson, pObj); - case QUERY_NODE_CREATE_TOPIC_STMT: - return jsonToCreateTopicStmt(pJson, pObj); - case QUERY_NODE_DROP_TOPIC_STMT: - return jsonToDropTopicStmt(pJson, pObj); - case QUERY_NODE_DROP_CGROUP_STMT: - return jsonToDropConsumerGroupStmt(pJson, pObj); - case QUERY_NODE_ALTER_LOCAL_STMT: - return jsonToAlterLocalStmt(pJson, pObj); - case QUERY_NODE_EXPLAIN_STMT: - return jsonToExplainStmt(pJson, pObj); - case QUERY_NODE_DESCRIBE_STMT: - return jsonToDescribeStmt(pJson, pObj); - case QUERY_NODE_COMPACT_DATABASE_STMT: - return jsonToCompactDatabaseStmt(pJson, pObj); - case QUERY_NODE_CREATE_STREAM_STMT: - return jsonToCreateStreamStmt(pJson, pObj); - case QUERY_NODE_DROP_STREAM_STMT: - return jsonToDropStreamStmt(pJson, pObj); - case QUERY_NODE_BALANCE_VGROUP_STMT: - return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize. - case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: - return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to deserialize. - case QUERY_NODE_MERGE_VGROUP_STMT: - return jsonToMergeVgroupStmt(pJson, pObj); - case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: - return jsonToRedistributeVgroupStmt(pJson, pObj); - case QUERY_NODE_SPLIT_VGROUP_STMT: - return jsonToSplitVgroupStmt(pJson, pObj); - case QUERY_NODE_GRANT_STMT: - return jsonToGrantStmt(pJson, pObj); - case QUERY_NODE_REVOKE_STMT: - return jsonToRevokeStmt(pJson, pObj); - case QUERY_NODE_SHOW_DNODES_STMT: - return jsonToShowDnodesStmt(pJson, pObj); - case QUERY_NODE_SHOW_MNODES_STMT: - return jsonToShowMnodesStmt(pJson, pObj); - case QUERY_NODE_SHOW_QNODES_STMT: - return jsonToShowQnodesStmt(pJson, pObj); - case QUERY_NODE_SHOW_CLUSTER_STMT: - return jsonToShowClusterStmt(pJson, pObj); - case QUERY_NODE_SHOW_DATABASES_STMT: - return jsonToShowDatabasesStmt(pJson, pObj); - case QUERY_NODE_SHOW_FUNCTIONS_STMT: - return jsonToShowFunctionsStmt(pJson, pObj); - case QUERY_NODE_SHOW_INDEXES_STMT: - return jsonToShowIndexesStmt(pJson, pObj); - case QUERY_NODE_SHOW_STABLES_STMT: - return jsonToShowStablesStmt(pJson, pObj); - case QUERY_NODE_SHOW_STREAMS_STMT: - return jsonToShowStreamsStmt(pJson, pObj); - case QUERY_NODE_SHOW_TABLES_STMT: - return jsonToShowTablesStmt(pJson, pObj); - case QUERY_NODE_SHOW_TAGS_STMT: - return jsonToShowTagsStmt(pJson, pObj); - case QUERY_NODE_SHOW_USERS_STMT: - return jsonToShowUsersStmt(pJson, pObj); - case QUERY_NODE_SHOW_VGROUPS_STMT: - return jsonToShowVgroupsStmt(pJson, pObj); - case QUERY_NODE_SHOW_CONSUMERS_STMT: - return jsonToShowConsumersStmt(pJson, pObj); - case QUERY_NODE_SHOW_VARIABLES_STMT: - return jsonToShowVariablesStmt(pJson, pObj); - case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: - return jsonToShowDnodeVariablesStmt(pJson, pObj); - case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - return jsonToShowTransactionsStmt(pJson, pObj); - case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: - return jsonToShowSubscriptionsStmt(pJson, pObj); - case QUERY_NODE_SHOW_VNODES_STMT: - return jsonToShowVnodesStmt(pJson, pObj); - case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: - return jsonToShowUserPrivilegesStmt(pJson, pObj); - case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: - return jsonToShowCreateDatabaseStmt(pJson, pObj); - case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - return jsonToShowCreateTableStmt(pJson, pObj); - case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - return jsonToShowCreateStableStmt(pJson, pObj); - case QUERY_NODE_SHOW_CREATE_VIEW_STMT: - return jsonToShowCreateViewStmt(pJson, pObj); - case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: - return jsonToShowTableDistributedStmt(pJson, pObj); - case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: - return jsonToShowLocalVariablesStmt(pJson, pObj); - case QUERY_NODE_SHOW_TABLE_TAGS_STMT: - return jsonToShowTableTagsStmt(pJson, pObj); - case QUERY_NODE_DELETE_STMT: - return jsonToDeleteStmt(pJson, pObj); - case QUERY_NODE_INSERT_STMT: - return jsonToInsertStmt(pJson, pObj); - case QUERY_NODE_RESTORE_DNODE_STMT: - return jsonToRestoreDnodeStmt(pJson, pObj); - case QUERY_NODE_RESTORE_QNODE_STMT: - return jsonToRestoreQnodeStmt(pJson, pObj); - case QUERY_NODE_RESTORE_MNODE_STMT: - return jsonToRestoreMnodeStmt(pJson, pObj); - case QUERY_NODE_RESTORE_VNODE_STMT: - return jsonToRestoreVnodeStmt(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_SCAN: - return jsonToLogicScanNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_JOIN: - return jsonToLogicJoinNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_AGG: - return jsonToLogicAggNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_PROJECT: - return jsonToLogicProjectNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: - return jsonToLogicVnodeModifyNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - return jsonToLogicExchangeNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_MERGE: - return jsonToLogicMergeNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return jsonToLogicWindowNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_FILL: - return jsonToLogicFillNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_SORT: - return jsonToLogicSortNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_PARTITION: - return jsonToLogicPartitionNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: - return jsonToLogicIndefRowsFuncNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: - return jsonToLogicInterpFuncNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: - return jsonToLogicGroupCacheNode(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL: - return jsonToLogicDynQueryCtrlNode(pJson, pObj); - case QUERY_NODE_LOGIC_SUBPLAN: - return jsonToLogicSubplan(pJson, pObj); - case QUERY_NODE_LOGIC_PLAN: - return jsonToLogicPlan(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - return jsonToPhysiTagScanNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: - return jsonToPhysiScanNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - return jsonToPhysiLastRowScanNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return jsonToPhysiTableScanNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - return jsonToPhysiSysTableScanNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - return jsonToPhysiProjectNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return jsonToPhysiMergeJoinNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: - return jsonToPhysiHashJoinNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: - return jsonToPhysiAggNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - return jsonToPhysiExchangeNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_MERGE: - return jsonToPhysiMergeNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_SORT: - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: - return jsonToPhysiSortNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - return jsonToPhysiIntervalNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_FILL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: - return jsonToPhysiFillNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - return jsonToPhysiSessionWindowNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: - return jsonToPhysiStateWindowNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: - return jsonToPhysiEventWindowNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: - return jsonToPhysiPartitionNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: - return jsonToPhysiStreamPartitionNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: - return jsonToPhysiIndefRowsFuncNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: - return jsonToPhysiInterpFuncNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - return jsonToPhysiDispatchNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: - return jsonToPhysiQueryInsertNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_DELETE: - return jsonToPhysiDeleteNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: - return jsonToPhysiGroupCacheNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: - return jsonToPhysiDynQueryCtrlNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_SUBPLAN: - return jsonToSubplan(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN: - return jsonToPlan(pJson, pObj); - default: - break; +int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { + ENodeType type = nodeType(pObj); + if (!funcArrayCheck(type)) { + return TSDB_CODE_SUCCESS; } - nodesWarn("jsonToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj))); + + if (funcNodes[type].toNodeFunc) { + return funcNodes[type].toNodeFunc(pJson, pObj); + } + + nodesWarn("jsonToSpecificNode unknown node type = %d", type); return TSDB_CODE_SUCCESS; } @@ -7436,3 +6673,2180 @@ int32_t nodesStringToList(const char* pStr, SNodeList** pList) { } return TSDB_CODE_SUCCESS; } + +static int32_t emptyNodeToJson(const void* pObj, SJson* pJson) { + return TSDB_CODE_SUCCESS; +} + +static int32_t emptyJsonToNode(const SJson* pJson, void* pObj) { + return TSDB_CODE_SUCCESS; +} + +static void destroyVgDataBlockArray(SArray* pArray) { + size_t size = taosArrayGetSize(pArray); + for (size_t i = 0; i < size; ++i) { + SVgDataBlocks* pVg = taosArrayGetP(pArray, i); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); + } + taosArrayDestroy(pArray); +} + +static void destroyLogicNode(SLogicNode* pNode) { + nodesDestroyList(pNode->pTargets); + nodesDestroyNode(pNode->pConditions); + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pLimit); + nodesDestroyNode(pNode->pSlimit); + nodesDestroyList(pNode->pHint); +} + +void destroyPhysiNode(SNode* pInput) { + SPhysiNode* pNode = (SPhysiNode*)pInput; + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pConditions); + nodesDestroyNode((SNode*)pNode->pOutputDataBlockDesc); + nodesDestroyNode(pNode->pLimit); + nodesDestroyNode(pNode->pSlimit); +} + +void destroyExprNode(SNode* pNode) { + SExprNode* pExpr = (SExprNode*)pNode; + taosArrayDestroy(pExpr->pAssociation); +} + +void destroyDataInSmaIndex(void* pIndex) { + taosMemoryFree(((STableIndexInfo*)pIndex)->expr); +} + +void destoryXNode(SNode* pNode) {} + +void destroyColumnNode(SNode* pNode) { + destroyExprNode(pNode); +} + +void destroyValueNode(SNode* pNode) { + SValueNode* pValue = (SValueNode*)pNode; + destroyExprNode(pNode); + taosMemoryFreeClear(pValue->literal); + if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + taosMemoryFreeClear(pValue->datum.p); + } +} + +void destroyOperatorNode(SNode* pNode) { + SOperatorNode* pOp = (SOperatorNode*)pNode; + destroyExprNode(pNode); + nodesDestroyNode(pOp->pLeft); + nodesDestroyNode(pOp->pRight); +} + +void destoryLogicConditionNode(SNode* pNode) { + destroyExprNode(pNode); + nodesDestroyList(((SLogicConditionNode*)pNode)->pParameterList); +} + +void destoryFunctionNode(SNode* pNode) { + destroyExprNode(pNode); + nodesDestroyList(((SFunctionNode*)pNode)->pParameterList); +} + +void destoryRealTableNode(SNode* pNode) { + SRealTableNode* pReal = (SRealTableNode*)pNode; + taosMemoryFreeClear(pReal->pMeta); + taosMemoryFreeClear(pReal->pVgroupList); + taosArrayDestroyEx(pReal->pSmaIndexes, destroyDataInSmaIndex); +} + +void destoryTempTableNode(SNode* pNode) { nodesDestroyNode(((STempTableNode*)pNode)->pSubquery); } + +void destoryJoinTableNode(SNode* pNode) { + SJoinTableNode* pJoin = (SJoinTableNode*)pNode; + nodesDestroyNode(pJoin->pLeft); + nodesDestroyNode(pJoin->pRight); + nodesDestroyNode(pJoin->pOnCond); +} + +void destoryGroupingSetNode(SNode* pNode) { nodesDestroyList(((SGroupingSetNode*)pNode)->pParameterList); } + +void destoryOrderByExprNode(SNode* pNode) { nodesDestroyNode(((SOrderByExprNode*)pNode)->pExpr); } + +void destoryStateWindowNode(SNode* pNode) { + SStateWindowNode* pState = (SStateWindowNode*)pNode; + nodesDestroyNode(pState->pCol); + nodesDestroyNode(pState->pExpr); +} + +void destorySessionWindowNode(SNode* pNode) { + SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; + nodesDestroyNode((SNode*)pSession->pCol); + nodesDestroyNode((SNode*)pSession->pGap); +} + +void destoryIntervalWindowNode(SNode* pNode) { + SIntervalWindowNode* pJoin = (SIntervalWindowNode*)pNode; + nodesDestroyNode(pJoin->pCol); + nodesDestroyNode(pJoin->pInterval); + nodesDestroyNode(pJoin->pOffset); + nodesDestroyNode(pJoin->pSliding); + nodesDestroyNode(pJoin->pFill); +} + +void destoryNodeListNode(SNode* pNode) { nodesDestroyList(((SNodeListNode*)pNode)->pNodeList); } + +void destoryFillNode(SNode* pNode) { + SFillNode* pFill = (SFillNode*)pNode; + nodesDestroyNode(pFill->pValues); + nodesDestroyNode(pFill->pWStartTs); +} + +void destoryRawExprNode(SNode* pNode) { nodesDestroyNode(((SRawExprNode*)pNode)->pNode); } + +void destoryTargetNode(SNode* pNode) { nodesDestroyNode(((STargetNode*)pNode)->pExpr); } + +void destoryDataBlockDescNode(SNode* pNode) { nodesDestroyList(((SDataBlockDescNode*)pNode)->pSlots); } + +void destoryDatabaseOptions(SNode* pNode) { + SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode; + nodesDestroyNode((SNode*)pOptions->pDaysPerFile); + nodesDestroyList(pOptions->pKeep); + nodesDestroyList(pOptions->pRetentions); +} + +void destoryTableOptions(SNode* pNode) { + STableOptions* pOptions = (STableOptions*)pNode; + nodesDestroyList(pOptions->pMaxDelay); + nodesDestroyList(pOptions->pWatermark); + nodesDestroyList(pOptions->pRollupFuncs); + nodesDestroyList(pOptions->pSma); + nodesDestroyList(pOptions->pDeleteMark); +} + +void destoryIndexOptions(SNode* pNode) { + SIndexOptions* pOptions = (SIndexOptions*)pNode; + nodesDestroyList(pOptions->pFuncs); + nodesDestroyNode(pOptions->pInterval); + nodesDestroyNode(pOptions->pOffset); + nodesDestroyNode(pOptions->pSliding); + nodesDestroyNode(pOptions->pStreamOptions); +} + +void destoryStreamOptions(SNode* pNode) { + SStreamOptions* pOptions = (SStreamOptions*)pNode; + nodesDestroyNode(pOptions->pDelay); + nodesDestroyNode(pOptions->pWatermark); + nodesDestroyNode(pOptions->pDeleteMark); +} + +void destoryWhenThenNode(SNode* pNode) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + nodesDestroyNode(pWhenThen->pWhen); + nodesDestroyNode(pWhenThen->pThen); +} + +void destoryCaseWhenNode(SNode* pNode) { + SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)pNode; + nodesDestroyNode(pCaseWhen->pCase); + nodesDestroyNode(pCaseWhen->pElse); + nodesDestroyList(pCaseWhen->pWhenThenList); +} + +void destoryEventWindowNode(SNode* pNode) { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + nodesDestroyNode(pEvent->pCol); + nodesDestroyNode(pEvent->pStartCond); + nodesDestroyNode(pEvent->pEndCond); +} + +void destoryHintNode(SNode* pNode) { + SHintNode* pHint = (SHintNode*)pNode; + taosMemoryFree(pHint->value); +} + +void destoryViewNode(SNode* pNode) { + SViewNode* pView = (SViewNode*)pNode; + taosMemoryFreeClear(pView->pMeta); + taosMemoryFreeClear(pView->pVgroupList); + taosArrayDestroyEx(pView->pSmaIndexes, destroyDataInSmaIndex); +} + +void destorySetOperator(SNode* pNode) { + SSetOperator* pStmt = (SSetOperator*)pNode; + nodesDestroyList(pStmt->pProjectionList); + nodesDestroyNode(pStmt->pLeft); + nodesDestroyNode(pStmt->pRight); + nodesDestroyList(pStmt->pOrderByList); + nodesDestroyNode(pStmt->pLimit); +} + +void destorySelectStmt(SNode* pNode) { + SSelectStmt* pStmt = (SSelectStmt*)pNode; + nodesDestroyList(pStmt->pProjectionList); + nodesDestroyNode(pStmt->pFromTable); + nodesDestroyNode(pStmt->pWhere); + nodesDestroyList(pStmt->pPartitionByList); + nodesDestroyList(pStmt->pTags); + nodesDestroyNode(pStmt->pSubtable); + nodesDestroyNode(pStmt->pWindow); + nodesDestroyList(pStmt->pGroupByList); + nodesDestroyNode(pStmt->pHaving); + nodesDestroyNode(pStmt->pRange); + nodesDestroyNode(pStmt->pEvery); + nodesDestroyNode(pStmt->pFill); + nodesDestroyList(pStmt->pOrderByList); + nodesDestroyNode((SNode*)pStmt->pLimit); + nodesDestroyNode((SNode*)pStmt->pSlimit); + nodesDestroyList(pStmt->pHint); +} + +void destoryVnodeModifyOpStmt(SNode* pNode) { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode; + destroyVgDataBlockArray(pStmt->pDataBlocks); + taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); + taosHashCleanup(pStmt->pVgroupsHashObj); + taosHashCleanup(pStmt->pSubTableHashObj); + taosHashCleanup(pStmt->pTableNameHashObj); + taosHashCleanup(pStmt->pDbFNameHashObj); + if (pStmt->freeHashFunc) { + pStmt->freeHashFunc(pStmt->pTableBlockHashObj); + } + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pStmt->pVgDataBlocks); + } + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); + if (pStmt->freeStbRowsCxtFunc) { + pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt); + } + taosMemoryFreeClear(pStmt->pStbRowsCxt); + taosCloseFile(&pStmt->fp); +} + +void destoryCreateDatabaseStmt(SNode* pNode) { nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions); } + +void destoryAlterDatabaseStmt(SNode* pNode) { nodesDestroyNode((SNode*)((SAlterDatabaseStmt*)pNode)->pOptions); } + +void destoryCreateTableStmt(SNode* pNode) { + SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode; + nodesDestroyList(pStmt->pCols); + nodesDestroyList(pStmt->pTags); + nodesDestroyNode((SNode*)pStmt->pOptions); +} + +void destoryCreateSubTableClause(SNode* pNode) { + SCreateSubTableClause* pStmt = (SCreateSubTableClause*)pNode; + nodesDestroyList(pStmt->pSpecificTags); + nodesDestroyList(pStmt->pValsOfTags); + nodesDestroyNode((SNode*)pStmt->pOptions); +} + +void destoryCreateMultiTablesStmt(SNode* pNode) { + nodesDestroyList(((SCreateMultiTablesStmt*)pNode)->pSubTables); +} + +void destoryDropTableStmt(SNode* pNode) { + nodesDestroyList(((SDropTableStmt*)pNode)->pTables); +} + +void destoryAlterTableStmt(SNode* pNode) { + SAlterTableStmt* pStmt = (SAlterTableStmt*)pNode; + nodesDestroyNode((SNode*)pStmt->pOptions); + nodesDestroyNode((SNode*)pStmt->pVal); +} + +void destoryCreateUserStmt(SNode* pNode) { + SCreateUserStmt* pStmt = (SCreateUserStmt*)pNode; + taosMemoryFree(pStmt->pIpRanges); + nodesDestroyList(pStmt->pNodeListIpRanges); +} + +void destoryAlterUserStmt(SNode* pNode) { + SAlterUserStmt* pStmt = (SAlterUserStmt*)pNode; + taosMemoryFree(pStmt->pIpRanges); + nodesDestroyList(pStmt->pNodeListIpRanges); +} + +void destoryCreateIndexStmt(SNode* pNode) { + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode; + nodesDestroyNode((SNode*)pStmt->pOptions); + nodesDestroyList(pStmt->pCols); + if (pStmt->pReq) { + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); + } +} + +void destoryCreateTopicStmt(SNode* pNode) { + nodesDestroyNode(((SCreateTopicStmt*)pNode)->pQuery); + nodesDestroyNode(((SCreateTopicStmt*)pNode)->pWhere); +} + +void destoryExplainStmt(SNode* pNode) { + SExplainStmt* pStmt = (SExplainStmt*)pNode; + nodesDestroyNode((SNode*)pStmt->pOptions); + nodesDestroyNode(pStmt->pQuery); +} + +void destoryDescribeStmt(SNode* pNode) { + taosMemoryFree(((SDescribeStmt*)pNode)->pMeta); +} + +void destoryCompactDatabaseStmt(SNode* pNode) { + SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)pNode; + nodesDestroyNode(pStmt->pStart); + nodesDestroyNode(pStmt->pEnd); +} + +void destoryCreateStreamStmt(SNode* pNode) { + SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pNode; + nodesDestroyNode((SNode*)pStmt->pOptions); + nodesDestroyNode(pStmt->pQuery); + nodesDestroyList(pStmt->pTags); + nodesDestroyNode(pStmt->pSubtable); + tFreeSCMCreateStreamReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); +} + +void destoryRedistributeVgroupStmt(SNode* pNode) { + nodesDestroyList(((SRedistributeVgroupStmt*)pNode)->pDnodes); +} + +void destoryGrantStmt(SNode* pNode) { + nodesDestroyNode(((SGrantStmt*)pNode)->pTagCond); +} + +void destoryRevokeStmt(SNode* pNode) { + nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond); +} + +void destoryShowStmt(SNode* pNode) { + SShowStmt* pStmt = (SShowStmt*)pNode; + nodesDestroyNode(pStmt->pDbName); + nodesDestroyNode(pStmt->pTbName); +} + +void destoryShowTableTagsStmt(SNode* pNode) { + SShowTableTagsStmt* pStmt = (SShowTableTagsStmt*)pNode; + nodesDestroyNode(pStmt->pDbName); + nodesDestroyNode(pStmt->pTbName); + nodesDestroyList(pStmt->pTags); +} + +void destoryShowDnodeVariablesStmt(SNode* pNode) { + nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pDnodeId); + nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pLikePattern); +} + +void destoryShowCreateDatabaseStmt(SNode* pNode) { + taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg); +} + +void destoryShowCreateTableStmt(SNode* pNode) { + STableCfg* pCfg = (STableCfg*)(((SShowCreateTableStmt*)pNode)->pTableCfg); + taosMemoryFreeClear(pCfg); + if (NULL == pCfg) { + return; + } + taosArrayDestroy(pCfg->pFuncs); + taosMemoryFree(pCfg->pComment); + taosMemoryFree(pCfg->pSchemas); + taosMemoryFree(pCfg->pTags); + taosMemoryFree(pCfg); +} + +void destoryDeleteStmt(SNode* pNode) { + SDeleteStmt* pStmt = (SDeleteStmt*)pNode; + nodesDestroyNode(pStmt->pFromTable); + nodesDestroyNode(pStmt->pWhere); + nodesDestroyNode(pStmt->pCountFunc); + nodesDestroyNode(pStmt->pFirstFunc); + nodesDestroyNode(pStmt->pLastFunc); + nodesDestroyNode(pStmt->pTagCond); +} + +void destoryInsertStmt(SNode* pNode) { + SInsertStmt* pStmt = (SInsertStmt*)pNode; + nodesDestroyNode(pStmt->pTable); + nodesDestroyList(pStmt->pCols); + nodesDestroyNode(pStmt->pQuery); +} + +void destoryQueryNode(SNode* pNode) { + SQuery* pQuery = (SQuery*)pNode; + nodesDestroyNode(pQuery->pPrevRoot); + nodesDestroyNode(pQuery->pRoot); + nodesDestroyNode(pQuery->pPostRoot); + taosMemoryFreeClear(pQuery->pResSchema); + if (NULL != pQuery->pCmdMsg) { + taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); + taosMemoryFreeClear(pQuery->pCmdMsg); + } + taosArrayDestroy(pQuery->pDbList); + taosArrayDestroy(pQuery->pTableList); + taosArrayDestroy(pQuery->pTargetTableList); + taosArrayDestroy(pQuery->pPlaceholderValues); + nodesDestroyNode(pQuery->pPrepareRoot); +} + +void destoryCreateViewStmt(SNode* pNode) { + SCreateViewStmt* pStmt = (SCreateViewStmt*)pNode; + taosMemoryFree(pStmt->pQuerySql); + tFreeSCMCreateViewReq(&pStmt->createReq); + nodesDestroyNode(pStmt->pQuery); +} + +void destoryScanLogicNode(SNode* pNode) { + SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pScanCols); + nodesDestroyList(pLogicNode->pScanPseudoCols); + taosMemoryFreeClear(pLogicNode->pVgroupList); + nodesDestroyList(pLogicNode->pDynamicScanFuncs); + nodesDestroyNode(pLogicNode->pTagCond); + nodesDestroyNode(pLogicNode->pTagIndexCond); + taosArrayDestroyEx(pLogicNode->pSmaIndexes, destroyDataInSmaIndex); + nodesDestroyList(pLogicNode->pGroupTags); + nodesDestroyList(pLogicNode->pTags); + nodesDestroyNode(pLogicNode->pSubtable); +} + +void destoryJoinLogicNode(SNode* pNode) { + SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyNode(pLogicNode->pPrimKeyEqCond); + nodesDestroyNode(pLogicNode->pOtherOnCond); + nodesDestroyNode(pLogicNode->pColEqCond); +} + +void destoryAggLogicNode(SNode* pNode) { + SAggLogicNode* pLogicNode = (SAggLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pAggFuncs); + nodesDestroyList(pLogicNode->pGroupKeys); +} + +void destoryProjectLogicNode(SNode* pNode) { + SProjectLogicNode* pLogicNode = (SProjectLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pProjections); +} + +void destoryVnodeModifyLogicNode(SNode* pNode) { + SVnodeModifyLogicNode* pLogicNode = (SVnodeModifyLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + destroyVgDataBlockArray(pLogicNode->pDataBlocks); + // pVgDataBlocks is weak reference + nodesDestroyNode(pLogicNode->pAffectedRows); + nodesDestroyNode(pLogicNode->pStartTs); + nodesDestroyNode(pLogicNode->pEndTs); + taosMemoryFreeClear(pLogicNode->pVgroupList); + nodesDestroyList(pLogicNode->pInsertCols); +} + +void destoryExchangeLogicNode(SNode* pNode) { + destroyLogicNode((SLogicNode*)pNode); +} + +void destoryMergeLogicNode(SNode* pNode) { + SMergeLogicNode* pLogicNode = (SMergeLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pMergeKeys); + nodesDestroyList(pLogicNode->pInputs); +} + +void destoryWindowLogicNode(SNode* pNode) { + SWindowLogicNode* pLogicNode = (SWindowLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pFuncs); + nodesDestroyNode(pLogicNode->pTspk); + nodesDestroyNode(pLogicNode->pTsEnd); + nodesDestroyNode(pLogicNode->pStateExpr); + nodesDestroyNode(pLogicNode->pStartCond); + nodesDestroyNode(pLogicNode->pEndCond); +} + +void destoryFillLogicNode(SNode* pNode) { + SFillLogicNode* pLogicNode = (SFillLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyNode(pLogicNode->pWStartTs); + nodesDestroyNode(pLogicNode->pValues); + nodesDestroyList(pLogicNode->pFillExprs); + nodesDestroyList(pLogicNode->pNotFillExprs); +} + +void destorySortLogicNode(SNode* pNode) { + SSortLogicNode* pLogicNode = (SSortLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pSortKeys); +} + +void destoryPartitionLogicNode(SNode* pNode) { + SPartitionLogicNode* pLogicNode = (SPartitionLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pPartitionKeys); + nodesDestroyList(pLogicNode->pTags); + nodesDestroyNode(pLogicNode->pSubtable); + nodesDestroyList(pLogicNode->pAggFuncs); +} + +void destoryIndefRowsFuncLogicNode(SNode* pNode) { + SIndefRowsFuncLogicNode* pLogicNode = (SIndefRowsFuncLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pFuncs); +} + +void destoryInterpFuncLogicNode(SNode* pNode) { + SInterpFuncLogicNode* pLogicNode = (SInterpFuncLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pFuncs); + nodesDestroyNode(pLogicNode->pFillValues); + nodesDestroyNode(pLogicNode->pTimeSeries); +} + +void destoryGroupCacheLogicNode(SNode* pNode) { + SGroupCacheLogicNode* pLogicNode = (SGroupCacheLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pGroupCols); +} + +void destoryDynQueryCtrlLogicNode(SNode* pNode) { + destroyLogicNode((SLogicNode*)pNode); +} + +void destoryLogicSubplan(SNode* pNode) { + SLogicSubplan* pSubplan = (SLogicSubplan*)pNode; + nodesDestroyList(pSubplan->pChildren); + nodesDestroyNode((SNode*)pSubplan->pNode); + nodesClearList(pSubplan->pParents); + taosMemoryFreeClear(pSubplan->pVgroupList); +} + +void destoryQueryLogicPlan(SNode* pNode) { + nodesDestroyList(((SQueryLogicPlan*)pNode)->pTopSubplans); +} + +void destroyScanPhysiNode(SNode* pInput) { + SScanPhysiNode* pNode = (SScanPhysiNode*)pInput; + destroyPhysiNode(pInput); + nodesDestroyList(pNode->pScanCols); + nodesDestroyList(pNode->pScanPseudoCols); +} + +void destoryLastRowScanPhysiNode(SNode* pNode) { + SLastRowScanPhysiNode* pPhyNode = (SLastRowScanPhysiNode*)pNode; + destroyScanPhysiNode(pNode); + nodesDestroyList(pPhyNode->pGroupTags); + nodesDestroyList(pPhyNode->pTargets); +} + +void destoryTableScanPhysiNode(SNode* pNode) { + STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; + destroyScanPhysiNode(pNode); + nodesDestroyList(pPhyNode->pDynamicScanFuncs); + nodesDestroyList(pPhyNode->pGroupTags); + nodesDestroyList(pPhyNode->pTags); + nodesDestroyNode(pPhyNode->pSubtable); +} + +void destoryProjectPhysiNode(SNode* pNode) { + SProjectPhysiNode* pPhyNode = (SProjectPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pProjections); +} + +void destorySortMergeJoinPhysiNode(SNode* pNode) { + SSortMergeJoinPhysiNode* pPhyNode = (SSortMergeJoinPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyNode(pPhyNode->pPrimKeyCond); + nodesDestroyNode(pPhyNode->pOtherOnCond); + nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pColEqCond); +} + +void destoryHashJoinPhysiNode(SNode* pNode) { + SHashJoinPhysiNode* pPhyNode = (SHashJoinPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pOnLeft); + nodesDestroyList(pPhyNode->pOnRight); + nodesDestroyNode(pPhyNode->pFilterConditions); + nodesDestroyList(pPhyNode->pTargets); + + nodesDestroyNode(pPhyNode->pPrimKeyCond); + nodesDestroyNode(pPhyNode->pColEqCond); + nodesDestroyNode(pPhyNode->pTagEqCond); +} + +void destoryAggPhysiNode(SNode* pNode) { + SAggPhysiNode* pPhyNode = (SAggPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pExprs); + nodesDestroyList(pPhyNode->pAggFuncs); + nodesDestroyList(pPhyNode->pGroupKeys); +} + +void destoryExchangePhysiNode(SNode* pNode) { + SExchangePhysiNode* pPhyNode = (SExchangePhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pSrcEndPoints); +} + +void destoryMergePhysiNode(SNode* pNode) { + SMergePhysiNode* pPhyNode = (SMergePhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pMergeKeys); + nodesDestroyList(pPhyNode->pTargets); +} + +void destorySortPhysiNode(SNode* pNode) { + SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pExprs); + nodesDestroyList(pPhyNode->pSortKeys); + nodesDestroyList(pPhyNode->pTargets); +} + +void destroyWindowPhysiNode(SNode* pInput) { + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pInput; + destroyPhysiNode(pInput); + nodesDestroyList(pNode->pExprs); + nodesDestroyList(pNode->pFuncs); + nodesDestroyNode(pNode->pTspk); + nodesDestroyNode(pNode->pTsEnd); +} + +void destoryFillPhysiNode(SNode* pNode) { + SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pFillExprs); + nodesDestroyList(pPhyNode->pNotFillExprs); + nodesDestroyNode(pPhyNode->pWStartTs); + nodesDestroyNode(pPhyNode->pValues); +} + +void destoryStateWindowPhysiNode(SNode* pNode) { + SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; + destroyWindowPhysiNode(pNode); + nodesDestroyNode(pPhyNode->pStateKey); +} + +void destoryEventWindowPhysiNode(SNode* pNode) { + SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; + destroyWindowPhysiNode(pNode); + nodesDestroyNode(pPhyNode->pStartCond); + nodesDestroyNode(pPhyNode->pEndCond); +} + +void destroyPartitionPhysiNode(SNode* pNode) { + SPartitionPhysiNode* pPartitionNode = (SPartitionPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPartitionNode->pExprs); + nodesDestroyList(pPartitionNode->pPartitionKeys); + nodesDestroyList(pPartitionNode->pTargets); +} + +void destoryStreamPartitionPhysiNode(SNode* pNode) { + SStreamPartitionPhysiNode* pPhyNode = (SStreamPartitionPhysiNode*)pNode; + destroyPartitionPhysiNode(pNode); + nodesDestroyList(pPhyNode->pTags); + nodesDestroyNode(pPhyNode->pSubtable); +} + +void destoryIndefRowsFuncPhysiNode(SNode* pNode) { + SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pExprs); + nodesDestroyList(pPhyNode->pFuncs); +} + +void destoryInterpFuncPhysiNode(SNode* pNode) { + SInterpFuncPhysiNode* pPhyNode = (SInterpFuncPhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pExprs); + nodesDestroyList(pPhyNode->pFuncs); + nodesDestroyNode(pPhyNode->pFillValues); + nodesDestroyNode(pPhyNode->pTimeSeries); +} + +void destroyDataSinkNode(SNode* pNode) { + SDataSinkNode* pDataNode = (SDataSinkNode*)pNode; + nodesDestroyNode((SNode*)pDataNode->pInputDataBlockDesc); +} + +void destoryDataInserterNode(SNode* pNode) { + SDataInserterNode* pSink = (SDataInserterNode*)pNode; + destroyDataSinkNode(pNode); + taosMemoryFreeClear(pSink->pData); +} + +void destoryQueryInserterNode(SNode* pNode) { + SQueryInserterNode* pSink = (SQueryInserterNode*)pNode; + destroyDataSinkNode(pNode); + nodesDestroyList(pSink->pCols); +} + +void destoryDataDeleterNode(SNode* pNode) { + SDataDeleterNode* pSink = (SDataDeleterNode*)pNode; + destroyDataSinkNode(pNode); + nodesDestroyNode(pSink->pAffectedRows); + nodesDestroyNode(pSink->pStartTs); + nodesDestroyNode(pSink->pEndTs); +} + +void destoryGroupCachePhysiNode(SNode* pNode) { + SGroupCachePhysiNode* pPhyNode = (SGroupCachePhysiNode*)pNode; + destroyPhysiNode(pNode); + nodesDestroyList(pPhyNode->pGroupCols); +} + +void destoryDynQueryCtrlPhysiNode(SNode* pNode) { + destroyPhysiNode(pNode); +} + +void destorySubplanNode(SNode* pNode) { + SSubplan* pSubplan = (SSubplan*)pNode; + nodesClearList(pSubplan->pChildren); + nodesDestroyNode((SNode*)pSubplan->pNode); + nodesDestroyNode((SNode*)pSubplan->pDataSink); + nodesDestroyNode((SNode*)pSubplan->pTagCond); + nodesDestroyNode((SNode*)pSubplan->pTagIndexCond); + nodesClearList(pSubplan->pParents); +} + +void destoryPlanNode(SNode* pNode) { + nodesDestroyList(((SQueryPlan*)pNode)->pSubplans); +} + +void nodesDestroyNode(SNode* pNode) { + if (NULL == pNode) { + return; + } + + int32_t index = nodeType(pNode); + if (!funcArrayCheck(index)) { + return; + } + if (funcNodes[index].destoryFunc) { + funcNodes[index].destoryFunc(pNode); + nodesFree(pNode); + return; + } + nodesError("nodesDestroyNode unknown node type = %d", nodeType(pNode)); + nodesFree(pNode); + return; +} + +// clang-format off +static void doInitNodeFuncArray() { + setFunc("Column", + QUERY_NODE_COLUMN, + sizeof(SColumnNode), + columnNodeToJson, + jsonToColumnNode, + destroyColumnNode + ); + setFunc("Value", + QUERY_NODE_VALUE, + sizeof(SValueNode), + valueNodeToJson, + jsonToValueNode, + destroyValueNode + ); + setFunc("Operator", + QUERY_NODE_OPERATOR, + sizeof(SOperatorNode), + operatorNodeToJson, + jsonToOperatorNode, + destroyOperatorNode + ); + setFunc("LogicCondition", + QUERY_NODE_LOGIC_CONDITION, + sizeof(SLogicConditionNode), + logicConditionNodeToJson, + jsonToLogicConditionNode, + destoryLogicConditionNode + ); + setFunc("Function", + QUERY_NODE_FUNCTION, + sizeof(SFunctionNode), + functionNodeToJson, + jsonToFunctionNode, + destoryFunctionNode + ); + setFunc("RealTable", + QUERY_NODE_REAL_TABLE, + sizeof(SRealTableNode), + realTableNodeToJson, + jsonToRealTableNode, + destoryRealTableNode + ); + setFunc("TempTable", + QUERY_NODE_TEMP_TABLE, + sizeof(STempTableNode), + tempTableNodeToJson, + jsonToTempTableNode, + destoryTempTableNode + ); + setFunc("JoinTable", + QUERY_NODE_JOIN_TABLE, + sizeof(SJoinTableNode), + joinTableNodeToJson, + jsonToJoinTableNode, + destoryJoinTableNode + ); + setFunc("GroupingSet", + QUERY_NODE_GROUPING_SET, + sizeof(SGroupingSetNode), + groupingSetNodeToJson, + jsonToGroupingSetNode, + destoryGroupingSetNode + ); + setFunc("OrderByExpr", + QUERY_NODE_ORDER_BY_EXPR, + sizeof(SOrderByExprNode), + orderByExprNodeToJson, + jsonToOrderByExprNode, + destoryOrderByExprNode + ); + setFunc("Limit", + QUERY_NODE_LIMIT, + sizeof(SLimitNode), + limitNodeToJson, + jsonToLimitNode, + destoryXNode + ); + setFunc("StateWindow", + QUERY_NODE_STATE_WINDOW, + sizeof(SStateWindowNode), + stateWindowNodeToJson, + jsonToStateWindowNode, + destoryStateWindowNode + ); + setFunc("SessionWinow", + QUERY_NODE_SESSION_WINDOW, + sizeof(SSessionWindowNode), + sessionWindowNodeToJson, + jsonToSessionWindowNode, + destorySessionWindowNode + ); + setFunc("IntervalWindow", + QUERY_NODE_INTERVAL_WINDOW, + sizeof(SIntervalWindowNode), + intervalWindowNodeToJson, + jsonToIntervalWindowNode, + destoryIntervalWindowNode + ); + setFunc("NodeList", + QUERY_NODE_NODE_LIST, + sizeof(SNodeListNode), + nodeListNodeToJson, + jsonToNodeListNode, + destoryNodeListNode + ); + setFunc("Fill", + QUERY_NODE_FILL, + sizeof(SFillNode), + fillNodeToJson, + jsonToFillNode, + destoryFillNode + ); + setFunc("RawExpr", + QUERY_NODE_RAW_EXPR, + sizeof(SRawExprNode), + emptyNodeToJson, + emptyJsonToNode, + destoryRawExprNode + ); + setFunc("Target", + QUERY_NODE_TARGET, + sizeof(STargetNode), + targetNodeToJson, + jsonToTargetNode, + destoryTargetNode + ); + setFunc("DataBlockDesc", + QUERY_NODE_DATABLOCK_DESC, + sizeof(SDataBlockDescNode), + dataBlockDescNodeToJson, + jsonToDataBlockDescNode, + destoryDataBlockDescNode + ); + setFunc("SlotDesc", + QUERY_NODE_SLOT_DESC, + sizeof(SSlotDescNode), + slotDescNodeToJson, + jsonToSlotDescNode, + destoryXNode + ); + setFunc("ColumnDef", + QUERY_NODE_COLUMN_DEF, + sizeof(SColumnDefNode), + columnDefNodeToJson, + jsonToColumnDefNode, + destoryXNode + ); + setFunc("DownstreamSource", + QUERY_NODE_DOWNSTREAM_SOURCE, + sizeof(SDownstreamSourceNode), + downstreamSourceNodeToJson, + jsonToDownstreamSourceNode, + destoryXNode + ); + setFunc("DatabaseOptions", + QUERY_NODE_DATABASE_OPTIONS, + sizeof(SDatabaseOptions), + databaseOptionsToJson, + jsonToDatabaseOptions, + destoryDatabaseOptions + ); + setFunc("TableOptions", + QUERY_NODE_TABLE_OPTIONS, + sizeof(STableOptions), + tableOptionsToJson, + jsonToTableOptions, + destoryTableOptions + ); + setFunc("IndexOptions", + QUERY_NODE_INDEX_OPTIONS, + sizeof(SIndexOptions), + indexOptionsToJson, + jsonToIndexOptions, + destoryIndexOptions + ); + setFunc("ExplainOptions", + QUERY_NODE_EXPLAIN_OPTIONS, + sizeof(SExplainOptions), + explainOptionsToJson, + jsonToExplainOptions, + destoryXNode + ); + setFunc("StreamOptions", + QUERY_NODE_STREAM_OPTIONS, + sizeof(SStreamOptions), + streamOptionsToJson, + jsonToStreamOptions, + destoryStreamOptions + ); + setFunc("LeftValue", + QUERY_NODE_LEFT_VALUE, + sizeof(SLeftValueNode), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("ColumnRef", + QUERY_NODE_COLUMN_REF, + sizeof(SColumnDefNode), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("WhenThen", + QUERY_NODE_WHEN_THEN, + sizeof(SWhenThenNode), + whenThenNodeToJson, + jsonToWhenThenNode, + destoryWhenThenNode + ); + setFunc("CaseWhen", + QUERY_NODE_CASE_WHEN, + sizeof(SCaseWhenNode), + caseWhenNodeToJson, + jsonToCaseWhenNode, + destoryCaseWhenNode + ); + setFunc("EventWindow", + QUERY_NODE_EVENT_WINDOW, + sizeof(SEventWindowNode), + eventWindowNodeToJson, + jsonToEventWindowNode, + destoryEventWindowNode + ); + setFunc("HintNode", + QUERY_NODE_HINT, + sizeof(SHintNode), + emptyNodeToJson, + emptyJsonToNode, + destoryHintNode + ); + setFunc("ViewNode", + QUERY_NODE_VIEW, + sizeof(SViewNode), + emptyNodeToJson, + emptyJsonToNode, + destoryViewNode + ); + setFunc("SetOperator", + QUERY_NODE_SET_OPERATOR, + sizeof(SSetOperator), + setOperatorToJson, + jsonToSetOperator, + destorySetOperator + ); + setFunc("SelectStmt", + QUERY_NODE_SELECT_STMT, + sizeof(SSelectStmt), + selectStmtToJson, + jsonToSelectStmt, + destorySelectStmt + ); + setFunc("VnodeModifyStmt", + QUERY_NODE_VNODE_MODIFY_STMT, + sizeof(SVnodeModifyOpStmt), + vnodeModifyStmtToJson, + jsonToVnodeModifyStmt, + destoryVnodeModifyOpStmt + ); + setFunc("CreateDatabaseStmt", + QUERY_NODE_CREATE_DATABASE_STMT, + sizeof(SCreateDatabaseStmt), + createDatabaseStmtToJson, + jsonToCreateDatabaseStmt, + destoryCreateDatabaseStmt + ); + setFunc("DropDatabaseStmt", + QUERY_NODE_DROP_DATABASE_STMT, + sizeof(SDropDatabaseStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("AlterDatabaseStmt", + QUERY_NODE_ALTER_DATABASE_STMT, + sizeof(SAlterDatabaseStmt), + alterDatabaseStmtToJson, + jsonToAlterDatabaseStmt, + destoryAlterDatabaseStmt + ); + setFunc("FlushDatabaseStmt", + QUERY_NODE_FLUSH_DATABASE_STMT, + sizeof(SFlushDatabaseStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("TrimDatabaseStmt", + QUERY_NODE_TRIM_DATABASE_STMT, + sizeof(STrimDatabaseStmt), + trimDatabaseStmtToJson, + jsonToTrimDatabaseStmt, + destoryXNode + ); + setFunc("CreateTableStmt", + QUERY_NODE_CREATE_TABLE_STMT, + sizeof(SCreateTableStmt), + createTableStmtToJson, + jsonToCreateTableStmt, + destoryCreateTableStmt + ); + setFunc("CreateSubtableClause", + QUERY_NODE_CREATE_SUBTABLE_CLAUSE, + sizeof(SCreateSubTableClause), + createSubTableClauseToJson, + jsonToCreateSubTableClause, + destoryCreateSubTableClause + ); + setFunc("CreateMultiTableStmt", + QUERY_NODE_CREATE_MULTI_TABLES_STMT, + sizeof(SCreateMultiTablesStmt), + createMultiTablesStmtToJson, + jsonToCreateMultiTablesStmt, + destoryCreateMultiTablesStmt + ); + setFunc("DropTableClause", + QUERY_NODE_DROP_TABLE_CLAUSE, + sizeof(SDropTableClause), + dropTableClauseToJson, + jsonToDropTableClause, + destoryXNode + ); + setFunc("DropTableStmt", + QUERY_NODE_DROP_TABLE_STMT, + sizeof(SDropTableStmt), + dropTableStmtToJson, + jsonToDropTableStmt, + destoryDropTableStmt + ); + setFunc("DropSuperTableStmt", + QUERY_NODE_DROP_SUPER_TABLE_STMT, + sizeof(SDropSuperTableStmt), + dropStableStmtToJson, + jsonToDropStableStmt, + destoryXNode + ); + setFunc("AlterTableStmt", + QUERY_NODE_ALTER_TABLE_STMT, + sizeof(SAlterTableStmt), + alterTableStmtToJson, + jsonToAlterTableStmt, + destoryAlterTableStmt + ); + setFunc("AlterSuperTableStmt", + QUERY_NODE_ALTER_SUPER_TABLE_STMT, + sizeof(SAlterTableStmt), + alterStableStmtToJson, + jsonToAlterStableStmt, + destoryAlterTableStmt + ); + setFunc("CreateUserStmt", + QUERY_NODE_CREATE_USER_STMT, + sizeof(SCreateUserStmt), + createUserStmtToJson, + jsonToCreateUserStmt, + destoryCreateUserStmt + ); + setFunc("AlterUserStmt", + QUERY_NODE_ALTER_USER_STMT, + sizeof(SAlterUserStmt), + alterUserStmtToJson, + jsonToAlterUserStmt, + destoryAlterUserStmt + ); + setFunc("DropUserStmt", + QUERY_NODE_DROP_USER_STMT, + sizeof(SDropUserStmt), + dropUserStmtToJson, + jsonToDropUserStmt, + destoryXNode + ); + setFunc("UseDatabaseStmt", + QUERY_NODE_USE_DATABASE_STMT, + sizeof(SUseDatabaseStmt), + useDatabaseStmtToJson, + jsonToUseDatabaseStmt, + destoryXNode + ); + setFunc("CreateDnodeStmt", + QUERY_NODE_CREATE_DNODE_STMT, + sizeof(SCreateDnodeStmt), + createDnodeStmtToJson, + jsonToCreateDnodeStmt, + destoryXNode + ); + setFunc("DropDnodeStmt", + QUERY_NODE_DROP_DNODE_STMT, + sizeof(SDropDnodeStmt), + dropDnodeStmtToJson, + jsonToDropDnodeStmt, + destoryXNode + ); + setFunc("AlterDnodeStmt", + QUERY_NODE_ALTER_DNODE_STMT, + sizeof(SAlterDnodeStmt), + alterDnodeStmtToJson, + jsonToAlterDnodeStmt, + destoryXNode + ); + setFunc("CreateIndexStmt", + QUERY_NODE_CREATE_INDEX_STMT, + sizeof(SCreateIndexStmt), + createIndexStmtToJson, + jsonToCreateIndexStmt, + destoryCreateIndexStmt + ); + setFunc("DropIndexStmt", + QUERY_NODE_DROP_INDEX_STMT, + sizeof(SDropIndexStmt), + dropIndexStmtToJson, + jsonToDropIndexStmt, + destoryXNode + ); + setFunc("CreateQnodeStmt", + QUERY_NODE_CREATE_QNODE_STMT, + sizeof(SCreateComponentNodeStmt), + createQnodeStmtToJson, + jsonToCreateQnodeStmt, + destoryXNode + ); + setFunc("DropQnodeStmt", + QUERY_NODE_DROP_QNODE_STMT, + sizeof(SDropComponentNodeStmt), + dropQnodeStmtToJson, + jsonToDropQnodeStmt, + destoryXNode + ); + setFunc("CreateBnodeStmt", + QUERY_NODE_CREATE_BNODE_STMT, + sizeof(SCreateComponentNodeStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("DropBnodeStmt", + QUERY_NODE_DROP_BNODE_STMT, + sizeof(SDropComponentNodeStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("CreateSnodeStmt", + QUERY_NODE_CREATE_SNODE_STMT, + sizeof(SCreateComponentNodeStmt), + createSnodeStmtToJson, + jsonToCreateSnodeStmt, + destoryXNode + ); + setFunc("DropSnodeStmt", + QUERY_NODE_DROP_SNODE_STMT, + sizeof(SDropComponentNodeStmt), + dropSnodeStmtToJson, + jsonToDropSnodeStmt, + destoryXNode + ); + setFunc("CreateMnodeStmt", + QUERY_NODE_CREATE_MNODE_STMT, + sizeof(SCreateComponentNodeStmt), + createMnodeStmtToJson, + jsonToCreateMnodeStmt, + destoryXNode + ); + setFunc("DropMnodeStmt", + QUERY_NODE_DROP_MNODE_STMT, + sizeof(SDropComponentNodeStmt), + dropMnodeStmtToJson, + jsonToDropMnodeStmt, + destoryXNode + ); + setFunc("CreateTopicStmt", + QUERY_NODE_CREATE_TOPIC_STMT, + sizeof(SCreateTopicStmt), + createTopicStmtToJson, + jsonToCreateTopicStmt, + destoryCreateTopicStmt + ); + setFunc("DropTopicStmt", + QUERY_NODE_DROP_TOPIC_STMT, + sizeof(SDropTopicStmt), + dropTopicStmtToJson, + jsonToDropTopicStmt, + destoryXNode + ); + setFunc("DropConsumerGroupStmt", + QUERY_NODE_DROP_CGROUP_STMT, + sizeof(SDropCGroupStmt), + dropConsumerGroupStmtToJson, + jsonToDropConsumerGroupStmt, + destoryXNode + ); + setFunc("AlterLocalStmt", + QUERY_NODE_ALTER_LOCAL_STMT, + sizeof(SAlterLocalStmt), + alterLocalStmtToJson, + jsonToAlterLocalStmt, + destoryXNode + ); + setFunc("ExplainStmt", + QUERY_NODE_EXPLAIN_STMT, + sizeof(SExplainStmt), + explainStmtToJson, + jsonToExplainStmt, + destoryExplainStmt + ); + setFunc("DescribeStmt", + QUERY_NODE_DESCRIBE_STMT, + sizeof(SDescribeStmt), + describeStmtToJson, + jsonToDescribeStmt, + destoryDescribeStmt + ); + setFunc("QueryCacheStmt", + QUERY_NODE_RESET_QUERY_CACHE_STMT, + sizeof(SNode), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("CompactDatabaseStmt", + QUERY_NODE_COMPACT_DATABASE_STMT, + sizeof(SCompactDatabaseStmt), + compactDatabaseStmtToJson, + jsonToCompactDatabaseStmt, + destoryCompactDatabaseStmt + ); + setFunc("CreateFunctionStmt", + QUERY_NODE_CREATE_FUNCTION_STMT, + sizeof(SCreateFunctionStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("DropFunctionStmt", + QUERY_NODE_DROP_FUNCTION_STMT, + sizeof(SDropFunctionStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("CreateStreamStmt", + QUERY_NODE_CREATE_STREAM_STMT, + sizeof(SCreateStreamStmt), + createStreamStmtToJson, + jsonToCreateStreamStmt, + destoryCreateStreamStmt + ); + setFunc("DropStreamStmt", + QUERY_NODE_DROP_STREAM_STMT, + sizeof(SDropStreamStmt), + dropStreamStmtToJson, + jsonToDropStreamStmt, + destoryXNode + ); + setFunc("PauseStreamStmt", + QUERY_NODE_PAUSE_STREAM_STMT, + sizeof(SPauseStreamStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("ResumeStreamStmt", + QUERY_NODE_RESUME_STREAM_STMT, + sizeof(SResumeStreamStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("BalanceVgroupStmt", + QUERY_NODE_BALANCE_VGROUP_STMT, + sizeof(SBalanceVgroupStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("BalanceVgroupLeaderStmt", + QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, + sizeof(SBalanceVgroupLeaderStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("MergeVgroupStmt", + QUERY_NODE_MERGE_VGROUP_STMT, + sizeof(SMergeVgroupStmt), + mergeVgroupStmtToJson, + jsonToMergeVgroupStmt, + destoryXNode + ); + setFunc("RedistributeVgroupStmt", + QUERY_NODE_REDISTRIBUTE_VGROUP_STMT, + sizeof(SRedistributeVgroupStmt), + redistributeVgroupStmtToJson, + jsonToRedistributeVgroupStmt, + destoryRedistributeVgroupStmt + ); + setFunc("SplitVgroupStmt", + QUERY_NODE_SPLIT_VGROUP_STMT, + sizeof(SSplitVgroupStmt), + splitVgroupStmtToJson, + jsonToSplitVgroupStmt, + destoryXNode + ); + setFunc("SyncDBStmt", + QUERY_NODE_SYNCDB_STMT, + 0, + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("GrantStmt", + QUERY_NODE_GRANT_STMT, + sizeof(SGrantStmt), + grantStmtToJson, + jsonToGrantStmt, + destoryGrantStmt + ); + setFunc("RevokeStmt", + QUERY_NODE_REVOKE_STMT, + sizeof(SRevokeStmt), + revokeStmtToJson, + jsonToRevokeStmt, + destoryRevokeStmt + ); + setFunc("ShowDnodesStmt", + QUERY_NODE_SHOW_DNODES_STMT, + sizeof(SShowStmt), + showDnodesStmtToJson, + jsonToShowDnodesStmt, + destoryShowStmt + ); + setFunc("ShowMnodesStmt", + QUERY_NODE_SHOW_MNODES_STMT, + sizeof(SShowStmt), + showMnodesStmtToJson, + jsonToShowMnodesStmt, + destoryShowStmt + ); + setFunc("ShowModulesStmt", + QUERY_NODE_SHOW_MODULES_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowQnodesStmt", + QUERY_NODE_SHOW_QNODES_STMT, + sizeof(SShowStmt), + showQnodesStmtToJson, + jsonToShowQnodesStmt, + destoryShowStmt + ); + setFunc("ShowSnodesStmt", + QUERY_NODE_SHOW_SNODES_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowBnodesStmt", + QUERY_NODE_SHOW_BNODES_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowClusterStmt", + QUERY_NODE_SHOW_CLUSTER_STMT, + sizeof(SShowStmt), + showClusterStmtToJson, + jsonToShowClusterStmt, + destoryShowStmt + ); + setFunc("ShowDatabaseStmt", + QUERY_NODE_SHOW_DATABASES_STMT, + sizeof(SShowStmt), + showDatabasesStmtToJson, + jsonToShowDatabasesStmt, + destoryShowStmt + ); + setFunc("ShowFunctionsStmt", + QUERY_NODE_SHOW_FUNCTIONS_STMT, + sizeof(SShowStmt), + showFunctionsStmtToJson, + jsonToShowFunctionsStmt, + destoryShowStmt + ); + setFunc("ShowIndexesStmt", + QUERY_NODE_SHOW_INDEXES_STMT, + sizeof(SShowStmt), + showIndexesStmtToJson, + jsonToShowIndexesStmt, + destoryShowStmt + ); + setFunc("ShowStablesStmt", + QUERY_NODE_SHOW_STABLES_STMT, + sizeof(SShowStmt), + showStablesStmtToJson, + jsonToShowStablesStmt, + destoryShowStmt + ); + setFunc("ShowStreamsStmt", + QUERY_NODE_SHOW_STREAMS_STMT, + sizeof(SShowStmt), + showStreamsStmtToJson, + jsonToShowStreamsStmt, + destoryShowStmt + ); + setFunc("ShowTablesStmt", + QUERY_NODE_SHOW_TABLES_STMT, + sizeof(SShowStmt), + showTablesStmtToJson, + jsonToShowTablesStmt, + destoryShowStmt + ); + setFunc("ShowTagsStmt", + QUERY_NODE_SHOW_TAGS_STMT, + sizeof(SShowStmt), + showTagsStmtToJson, + jsonToShowTagsStmt, + destoryShowStmt + ); + setFunc("ShowUsersStmt", + QUERY_NODE_SHOW_USERS_STMT, + sizeof(SShowStmt), + showUsersStmtToJson, + jsonToShowUsersStmt, + destoryShowStmt + ); + setFunc("ShowLicencesStmt", + QUERY_NODE_SHOW_LICENCES_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowVgroupsStmt", + QUERY_NODE_SHOW_VGROUPS_STMT, + sizeof(SShowStmt), + showVgroupsStmtToJson, + jsonToShowVgroupsStmt, + destoryShowStmt + ); + setFunc("ShowTopicsStmt", + QUERY_NODE_SHOW_TOPICS_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowConsumersStmt", + QUERY_NODE_SHOW_CONSUMERS_STMT, + sizeof(SShowStmt), + showConsumersStmtToJson, + jsonToShowConsumersStmt, + destoryShowStmt + ); + setFunc("ShowQueriesStmt", + QUERY_NODE_SHOW_QUERIES_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowConnectionsStmt", + QUERY_NODE_SHOW_CONNECTIONS_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowAppsStmt", + QUERY_NODE_SHOW_APPS_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowVariablesStmt", + QUERY_NODE_SHOW_VARIABLES_STMT, + sizeof(SShowStmt), + showVariablesStmtToJson, + jsonToShowVariablesStmt, + destoryShowStmt + ); + setFunc("ShowDnodeVariablesStmt", + QUERY_NODE_SHOW_DNODE_VARIABLES_STMT, + sizeof(SShowDnodeVariablesStmt), + showDnodeVariablesStmtToJson, + jsonToShowDnodeVariablesStmt, + destoryShowDnodeVariablesStmt + ); + setFunc("ShowTransactionsStmt", + QUERY_NODE_SHOW_TRANSACTIONS_STMT, + sizeof(SShowStmt), + showTransactionsStmtToJson, + jsonToShowTransactionsStmt, + destoryShowStmt + ); + setFunc("ShowSubscriptionsStmt", + QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, + sizeof(SShowStmt), + showSubscriptionsStmtToJson, + jsonToShowSubscriptionsStmt, + destoryShowStmt + ); + setFunc("ShowVnodeStmt", + QUERY_NODE_SHOW_VNODES_STMT, + sizeof(SShowStmt), + showVnodesStmtToJson, + jsonToShowVnodesStmt, + destoryShowStmt + ); + setFunc("ShowUserPrivilegesStmt", + QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, + sizeof(SShowStmt), + showUserPrivilegesStmtToJson, + jsonToShowUserPrivilegesStmt, + destoryShowStmt + ); + setFunc("ShowViewsStmt", + QUERY_NODE_SHOW_VIEWS_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowCreateViewStmt", + QUERY_NODE_SHOW_CREATE_VIEW_STMT, + sizeof(SShowCreateViewStmt), + showCreateViewStmtToJson, + jsonToShowCreateViewStmt, + destoryXNode + ); + setFunc("ShowCreateDatabasesStmt", + QUERY_NODE_SHOW_CREATE_DATABASE_STMT, + sizeof(SShowCreateDatabaseStmt), + showCreateDatabaseStmtToJson, + jsonToShowCreateDatabaseStmt, + destoryShowCreateDatabaseStmt + ); + setFunc("ShowCreateTablesStmt", + QUERY_NODE_SHOW_CREATE_TABLE_STMT, + sizeof(SShowCreateTableStmt), + showCreateTableStmtToJson, + jsonToShowCreateTableStmt, + destoryShowCreateTableStmt + ); + setFunc("ShowCreateStablesStmt", + QUERY_NODE_SHOW_CREATE_STABLE_STMT, + sizeof(SShowCreateTableStmt), + showCreateStableStmtToJson, + jsonToShowCreateStableStmt, + destoryShowCreateTableStmt + ); + setFunc("ShowTableDistributedStmt", + QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, + sizeof(SShowTableDistributedStmt), + showTableDistributedStmtToJson, + jsonToShowTableDistributedStmt, + destoryXNode + ); + setFunc("ShowLocalVariablesStmt", + QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, + sizeof(SShowStmt), + showLocalVariablesStmtToJson, + jsonToShowLocalVariablesStmt, + destoryShowStmt + ); + setFunc("ShowScoresStmt", + QUERY_NODE_SHOW_SCORES_STMT, + sizeof(SShowStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryShowStmt + ); + setFunc("ShowTableTagsStmt", + QUERY_NODE_SHOW_TABLE_TAGS_STMT, + sizeof(SShowTableTagsStmt), + showTableTagsStmtToJson, + jsonToShowTableTagsStmt, + destoryShowTableTagsStmt + ); + setFunc("KillConnectionStmt", + QUERY_NODE_KILL_CONNECTION_STMT, + sizeof(SKillStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("KillQueryStmt", + QUERY_NODE_KILL_QUERY_STMT, + sizeof(SKillQueryStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("KillTransactionStmt", + QUERY_NODE_KILL_TRANSACTION_STMT, + sizeof(SKillStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("DeleteStmt", + QUERY_NODE_DELETE_STMT, + sizeof(SDeleteStmt), + deleteStmtToJson, + jsonToDeleteStmt, + destoryDeleteStmt + ); + setFunc("InsertStmt", + QUERY_NODE_INSERT_STMT, + sizeof(SInsertStmt), + insertStmtToJson, + jsonToInsertStmt, + destoryInsertStmt + ); + setFunc("QueryNode", + QUERY_NODE_QUERY, + sizeof(SQuery), + emptyNodeToJson, + emptyJsonToNode, + destoryQueryNode + ); + setFunc("ShowDbAliveStmt", + QUERY_NODE_SHOW_DB_ALIVE_STMT, + sizeof(SShowAliveStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("ShowClusterAliveStmt", + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, + sizeof(SShowAliveStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("RestoreDnodeStmt", + QUERY_NODE_RESTORE_DNODE_STMT, + sizeof(SRestoreComponentNodeStmt), + emptyNodeToJson, + jsonToRestoreDnodeStmt, + destoryXNode + ); + setFunc("RestoreQnodeStmt", + QUERY_NODE_RESTORE_QNODE_STMT, + sizeof(SRestoreComponentNodeStmt), + emptyNodeToJson, + jsonToRestoreQnodeStmt, + destoryXNode + ); + setFunc("RestoreMnodeStmt", + QUERY_NODE_RESTORE_MNODE_STMT, + sizeof(SRestoreComponentNodeStmt), + emptyNodeToJson, + jsonToRestoreMnodeStmt, + destoryXNode + ); + setFunc("RestoreVnodeStmt", + QUERY_NODE_RESTORE_VNODE_STMT, + sizeof(SRestoreComponentNodeStmt), + emptyNodeToJson, + jsonToRestoreVnodeStmt, + destoryXNode + ); + setFunc("CreateViewStmt", + QUERY_NODE_CREATE_VIEW_STMT, + sizeof(SCreateViewStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryCreateViewStmt + ); + setFunc("DropViewStmt", + QUERY_NODE_DROP_VIEW_STMT, + sizeof(SDropViewStmt), + emptyNodeToJson, + emptyJsonToNode, + destoryXNode + ); + setFunc("LogicScan", + QUERY_NODE_LOGIC_PLAN_SCAN, + sizeof(SScanLogicNode), + logicScanNodeToJson, + jsonToLogicScanNode, + destoryScanLogicNode + ); + setFunc("LogicJoin", + QUERY_NODE_LOGIC_PLAN_JOIN, + sizeof(SJoinLogicNode), + logicJoinNodeToJson, + jsonToLogicJoinNode, + destoryJoinLogicNode + ); + setFunc("LogicAgg", + QUERY_NODE_LOGIC_PLAN_AGG, + sizeof(SAggLogicNode), + logicAggNodeToJson, + jsonToLogicAggNode, + destoryAggLogicNode + ); + setFunc("LogicProject", + QUERY_NODE_LOGIC_PLAN_PROJECT, + sizeof(SProjectLogicNode), + logicProjectNodeToJson, + jsonToLogicProjectNode, + destoryProjectLogicNode + ); + setFunc("LogicVnodeModify", + QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY, + sizeof(SVnodeModifyLogicNode), + logicVnodeModifyNodeToJson, + jsonToLogicVnodeModifyNode, + destoryVnodeModifyLogicNode + ); + setFunc("LogicExchange", + QUERY_NODE_LOGIC_PLAN_EXCHANGE, + sizeof(SExchangeLogicNode), + logicExchangeNodeToJson, + jsonToLogicExchangeNode, + destoryExchangeLogicNode + ); + setFunc("LogicMerge", + QUERY_NODE_LOGIC_PLAN_MERGE, + sizeof(SMergeLogicNode), + logicMergeNodeToJson, + jsonToLogicMergeNode, + destoryMergeLogicNode + ); + setFunc("LogicWindow", + QUERY_NODE_LOGIC_PLAN_WINDOW, + sizeof(SWindowLogicNode), + logicWindowNodeToJson, + jsonToLogicWindowNode, + destoryWindowLogicNode + ); + setFunc("LogicFill", + QUERY_NODE_LOGIC_PLAN_FILL, + sizeof(SFillLogicNode), + logicFillNodeToJson, + jsonToLogicFillNode, + destoryFillLogicNode + ); + setFunc("LogicSort", + QUERY_NODE_LOGIC_PLAN_SORT, + sizeof(SSortLogicNode), + logicSortNodeToJson, + jsonToLogicSortNode, + destorySortLogicNode + ); + setFunc("LogicPartition", + QUERY_NODE_LOGIC_PLAN_PARTITION, + sizeof(SPartitionLogicNode), + logicPartitionNodeToJson, + jsonToLogicPartitionNode, + destoryPartitionLogicNode + ); + setFunc("LogicIndefRowsFunc", + QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC, + sizeof(SIndefRowsFuncLogicNode), + logicIndefRowsFuncNodeToJson, + jsonToLogicIndefRowsFuncNode, + destoryIndefRowsFuncLogicNode + ); + setFunc("LogicInterpFunc", + QUERY_NODE_LOGIC_PLAN_INTERP_FUNC, + sizeof(SInterpFuncLogicNode), + logicInterpFuncNodeToJson, + jsonToLogicInterpFuncNode, + destoryInterpFuncLogicNode + ); + setFunc("LogicGroupCache", + QUERY_NODE_LOGIC_PLAN_GROUP_CACHE, + sizeof(SGroupCacheLogicNode), + logicGroupCacheNodeToJson, + jsonToLogicGroupCacheNode, + destoryGroupCacheLogicNode + ); + setFunc("LogicDynamicQueryCtrl", + QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL, + sizeof(SDynQueryCtrlLogicNode), + logicDynQueryCtrlNodeToJson, + jsonToLogicDynQueryCtrlNode, + destoryDynQueryCtrlLogicNode + ); + setFunc("LogicSubplan", + QUERY_NODE_LOGIC_SUBPLAN, + sizeof(SLogicSubplan), + logicSubplanToJson, + jsonToLogicSubplan, + destoryLogicSubplan + ); + setFunc("LogicPlan", + QUERY_NODE_LOGIC_PLAN, + sizeof(SQueryLogicPlan), + logicPlanToJson, + jsonToLogicPlan, + destoryQueryLogicPlan + ); + setFunc("PhysiTagScan", + QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, + sizeof(STagScanPhysiNode), + physiTagScanNodeToJson, + jsonToPhysiTagScanNode, + destroyScanPhysiNode + ); + setFunc("PhysiTableScan", + QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, + sizeof(STableScanPhysiNode), + physiTableScanNodeToJson, + jsonToPhysiTableScanNode, + destoryTableScanPhysiNode + ); + setFunc("PhysiTableSeqScan", + QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, + sizeof(STableSeqScanPhysiNode), + emptyNodeToJson, + emptyJsonToNode, + destoryTableScanPhysiNode + ); + setFunc("PhysiTableMergeScan", + QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, + sizeof(STableMergeScanPhysiNode), + physiTableScanNodeToJson, + jsonToPhysiTableScanNode, + destoryTableScanPhysiNode + ); + setFunc("PhysiSreamScan", + QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, + sizeof(SStreamScanPhysiNode), + physiTableScanNodeToJson, + jsonToPhysiTableScanNode, + destoryTableScanPhysiNode + ); + setFunc("PhysiSystemTableScan", + QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, + sizeof(SSystemTableScanPhysiNode), + physiSysTableScanNodeToJson, + jsonToPhysiSysTableScanNode, + destroyScanPhysiNode + ); + setFunc("PhysiBlockDistScan", + QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, + sizeof(SBlockDistScanPhysiNode), + physiScanNodeToJson, + jsonToPhysiScanNode, + destroyScanPhysiNode + ); + setFunc("PhysiLastRowScan", + QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, + sizeof(SLastRowScanPhysiNode), + physiLastRowScanNodeToJson, + jsonToPhysiLastRowScanNode, + destoryLastRowScanPhysiNode + ); + setFunc("PhysiProject", + QUERY_NODE_PHYSICAL_PLAN_PROJECT, + sizeof(SProjectPhysiNode), + physiProjectNodeToJson, + jsonToPhysiProjectNode, + destoryProjectPhysiNode + ); + setFunc("PhysiMergeJoin", + QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, + sizeof(SSortMergeJoinPhysiNode), + physiMergeJoinNodeToJson, + jsonToPhysiMergeJoinNode, + destorySortMergeJoinPhysiNode + ); + setFunc("PhysiAgg", + QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, + sizeof(SAggPhysiNode), + physiAggNodeToJson, + jsonToPhysiAggNode, + destoryAggPhysiNode + ); + setFunc("PhysiExchange", + QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, + sizeof(SExchangePhysiNode), + physiExchangeNodeToJson, + jsonToPhysiExchangeNode, + destoryExchangePhysiNode + ); + setFunc("PhysiMerge", + QUERY_NODE_PHYSICAL_PLAN_MERGE, + sizeof(SMergePhysiNode), + physiMergeNodeToJson, + jsonToPhysiMergeNode, + destoryMergePhysiNode + ); + setFunc("PhysiSort", + QUERY_NODE_PHYSICAL_PLAN_SORT, + sizeof(SSortPhysiNode), + physiSortNodeToJson, + jsonToPhysiSortNode, + destorySortPhysiNode + ); + setFunc("PhysiGroupSort", + QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, + sizeof(SGroupSortPhysiNode), + physiSortNodeToJson, + jsonToPhysiSortNode, + destorySortPhysiNode + ); + setFunc("PhysiHashInterval", + QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, + sizeof(SIntervalPhysiNode), + physiIntervalNodeToJson, + jsonToPhysiIntervalNode, + destroyWindowPhysiNode + ); + setFunc("PhysiMergeAlignedInterval", + QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, + sizeof(SMergeAlignedIntervalPhysiNode), + physiIntervalNodeToJson, + jsonToPhysiIntervalNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStreamInterval", + QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, + sizeof(SStreamIntervalPhysiNode), + physiIntervalNodeToJson, + jsonToPhysiIntervalNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStreamFinalInterval", + QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, + sizeof(SStreamFinalIntervalPhysiNode), + physiIntervalNodeToJson, + jsonToPhysiIntervalNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStreamSemiInterval", + QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, + sizeof(SStreamSemiIntervalPhysiNode), + physiIntervalNodeToJson, + jsonToPhysiIntervalNode, + destroyWindowPhysiNode + ); + setFunc("PhysiFill", + QUERY_NODE_PHYSICAL_PLAN_FILL, + sizeof(SFillPhysiNode), + physiFillNodeToJson, + jsonToPhysiFillNode, + destoryFillPhysiNode + ); + setFunc("PhysiStreamFill", + QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, + sizeof(SFillPhysiNode), + physiFillNodeToJson, + jsonToPhysiFillNode, + destoryFillPhysiNode + ); + setFunc("PhysiSessionWindow", + QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, + sizeof(SSessionWinodwPhysiNode), + physiSessionWindowNodeToJson, + jsonToPhysiSessionWindowNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStreamSessionWindow", + QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, + sizeof(SStreamSessionWinodwPhysiNode), + physiSessionWindowNodeToJson, + jsonToPhysiSessionWindowNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStreamSemiSessionWindow", + QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, + sizeof(SStreamSemiSessionWinodwPhysiNode), + physiSessionWindowNodeToJson, + jsonToPhysiSessionWindowNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStreamFinalSessionWindow", + QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, + sizeof(SStreamFinalSessionWinodwPhysiNode), + physiSessionWindowNodeToJson, + jsonToPhysiSessionWindowNode, + destroyWindowPhysiNode + ); + setFunc("PhysiStateWindow", + QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, + sizeof(SStateWinodwPhysiNode), + physiStateWindowNodeToJson, + jsonToPhysiStateWindowNode, + destoryStateWindowPhysiNode + ); + setFunc("PhysiStreamStateWindow", + QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, + sizeof(SStreamStateWinodwPhysiNode), + physiStateWindowNodeToJson, + jsonToPhysiStateWindowNode, + destoryStateWindowPhysiNode + ); + setFunc("PhysiPartition", + QUERY_NODE_PHYSICAL_PLAN_PARTITION, + sizeof(SPartitionPhysiNode), + physiPartitionNodeToJson, + jsonToPhysiPartitionNode, + destroyPartitionPhysiNode + ); + setFunc("PhysiStreamPartition", + QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, + sizeof(SStreamPartitionPhysiNode), + physiStreamPartitionNodeToJson, + jsonToPhysiStreamPartitionNode, + destoryStreamPartitionPhysiNode + ); + setFunc("PhysiIndefRowsFunc", + QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, + sizeof(SIndefRowsFuncPhysiNode), + physiIndefRowsFuncNodeToJson, + jsonToPhysiIndefRowsFuncNode, + destoryIndefRowsFuncPhysiNode + ); + setFunc("PhysiInterpFunc", + QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, + sizeof(SInterpFuncLogicNode), + physiInterpFuncNodeToJson, + jsonToPhysiInterpFuncNode, + destoryInterpFuncPhysiNode + ); + setFunc("PhysiDispatch", + QUERY_NODE_PHYSICAL_PLAN_DISPATCH, + sizeof(SDataDispatcherNode), + physiDispatchNodeToJson, + jsonToPhysiDispatchNode, + destroyDataSinkNode + ); + setFunc("PhysiInsert", + QUERY_NODE_PHYSICAL_PLAN_INSERT, + sizeof(SDataInserterNode), + emptyNodeToJson, + emptyJsonToNode, + destoryDataInserterNode + ); + setFunc("PhysiQueryInsert", + QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, + sizeof(SQueryInserterNode), + physiQueryInsertNodeToJson, + jsonToPhysiQueryInsertNode, + destoryQueryInserterNode + ); + setFunc("PhysiDelete", + QUERY_NODE_PHYSICAL_PLAN_DELETE, + sizeof(SDataDeleterNode), + physiDeleteNodeToJson, + jsonToPhysiDeleteNode, + destoryDataDeleterNode + ); + setFunc("PhysiGroupCache", + QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, + sizeof(SGroupCachePhysiNode), + physiGroupCacheNodeToJson, + jsonToPhysiGroupCacheNode, + destoryGroupCachePhysiNode + ); + setFunc("PhysiDynamicQueryCtrl", + QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, + sizeof(SDynQueryCtrlPhysiNode), + physiDynQueryCtrlNodeToJson, + jsonToPhysiDynQueryCtrlNode, + destoryDynQueryCtrlPhysiNode + ); + setFunc("PhysiSubplan", + QUERY_NODE_PHYSICAL_SUBPLAN, + sizeof(SSubplan), + subplanToJson, + jsonToSubplan, + destorySubplanNode + ); + setFunc("PhysiPlan", + QUERY_NODE_PHYSICAL_PLAN, + sizeof(SQueryPlan), + planToJson, + jsonToPlan, + destoryPlanNode + ); + setFunc("PhysiTableCountScan", + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, + sizeof(STableCountScanPhysiNode), + physiLastRowScanNodeToJson, + jsonToPhysiScanNode, + destoryLastRowScanPhysiNode + ); + setFunc("PhysiMergeEventWindow", + QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, + sizeof(SEventWinodwPhysiNode), + physiEventWindowNodeToJson, + jsonToPhysiEventWindowNode, + destoryEventWindowPhysiNode + ); + setFunc("PhysiStreamEventWindow", + QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, + sizeof(SStreamEventWinodwPhysiNode), + physiEventWindowNodeToJson, + jsonToPhysiEventWindowNode, + destoryEventWindowPhysiNode + ); + setFunc("PhysiHashJoin", + QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, + sizeof(SHashJoinPhysiNode), + physiHashJoinNodeToJson, + jsonToPhysiHashJoinNode, + destoryHashJoinPhysiNode + ); + initNodeCode = 0; +} + +// clang-format on diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index c8de8bdfac..3a738dbbd2 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -131,7 +131,7 @@ static void destroyNodeAllocator(void* p) { } int32_t nodesInitAllocatorSet() { - initNodeName(); + nodesInit(); if (g_allocatorReqRefPool >= 0) { nodesWarn("nodes already initialized"); return TSDB_CODE_SUCCESS; @@ -238,1245 +238,14 @@ static SNode* makeNode(ENodeType type, int32_t size) { } SNode* nodesMakeNode(ENodeType type) { - switch (type) { - case QUERY_NODE_COLUMN: - return makeNode(type, sizeof(SColumnNode)); - case QUERY_NODE_VALUE: - return makeNode(type, sizeof(SValueNode)); - case QUERY_NODE_OPERATOR: - return makeNode(type, sizeof(SOperatorNode)); - case QUERY_NODE_LOGIC_CONDITION: - return makeNode(type, sizeof(SLogicConditionNode)); - case QUERY_NODE_FUNCTION: - return makeNode(type, sizeof(SFunctionNode)); - case QUERY_NODE_REAL_TABLE: - return makeNode(type, sizeof(SRealTableNode)); - case QUERY_NODE_TEMP_TABLE: - return makeNode(type, sizeof(STempTableNode)); - case QUERY_NODE_JOIN_TABLE: - return makeNode(type, sizeof(SJoinTableNode)); - case QUERY_NODE_GROUPING_SET: - return makeNode(type, sizeof(SGroupingSetNode)); - case QUERY_NODE_ORDER_BY_EXPR: - return makeNode(type, sizeof(SOrderByExprNode)); - case QUERY_NODE_LIMIT: - return makeNode(type, sizeof(SLimitNode)); - case QUERY_NODE_STATE_WINDOW: - return makeNode(type, sizeof(SStateWindowNode)); - case QUERY_NODE_SESSION_WINDOW: - return makeNode(type, sizeof(SSessionWindowNode)); - case QUERY_NODE_INTERVAL_WINDOW: - return makeNode(type, sizeof(SIntervalWindowNode)); - case QUERY_NODE_NODE_LIST: - return makeNode(type, sizeof(SNodeListNode)); - case QUERY_NODE_FILL: - return makeNode(type, sizeof(SFillNode)); - case QUERY_NODE_RAW_EXPR: - return makeNode(type, sizeof(SRawExprNode)); - case QUERY_NODE_TARGET: - return makeNode(type, sizeof(STargetNode)); - case QUERY_NODE_DATABLOCK_DESC: - return makeNode(type, sizeof(SDataBlockDescNode)); - case QUERY_NODE_SLOT_DESC: - return makeNode(type, sizeof(SSlotDescNode)); - case QUERY_NODE_COLUMN_DEF: - return makeNode(type, sizeof(SColumnDefNode)); - case QUERY_NODE_DOWNSTREAM_SOURCE: - return makeNode(type, sizeof(SDownstreamSourceNode)); - case QUERY_NODE_DATABASE_OPTIONS: - return makeNode(type, sizeof(SDatabaseOptions)); - case QUERY_NODE_TABLE_OPTIONS: - return makeNode(type, sizeof(STableOptions)); - case QUERY_NODE_INDEX_OPTIONS: - return makeNode(type, sizeof(SIndexOptions)); - case QUERY_NODE_EXPLAIN_OPTIONS: - return makeNode(type, sizeof(SExplainOptions)); - case QUERY_NODE_STREAM_OPTIONS: - return makeNode(type, sizeof(SStreamOptions)); - case QUERY_NODE_LEFT_VALUE: - return makeNode(type, sizeof(SLeftValueNode)); - case QUERY_NODE_COLUMN_REF: - return makeNode(type, sizeof(SColumnDefNode)); - case QUERY_NODE_WHEN_THEN: - return makeNode(type, sizeof(SWhenThenNode)); - case QUERY_NODE_CASE_WHEN: - return makeNode(type, sizeof(SCaseWhenNode)); - case QUERY_NODE_EVENT_WINDOW: - return makeNode(type, sizeof(SEventWindowNode)); - case QUERY_NODE_HINT: - return makeNode(type, sizeof(SHintNode)); - case QUERY_NODE_VIEW: - return makeNode(type, sizeof(SViewNode)); - case QUERY_NODE_SET_OPERATOR: - return makeNode(type, sizeof(SSetOperator)); - case QUERY_NODE_SELECT_STMT: - return makeNode(type, sizeof(SSelectStmt)); - case QUERY_NODE_VNODE_MODIFY_STMT: - return makeNode(type, sizeof(SVnodeModifyOpStmt)); - case QUERY_NODE_CREATE_DATABASE_STMT: - return makeNode(type, sizeof(SCreateDatabaseStmt)); - case QUERY_NODE_DROP_DATABASE_STMT: - return makeNode(type, sizeof(SDropDatabaseStmt)); - case QUERY_NODE_ALTER_DATABASE_STMT: - return makeNode(type, sizeof(SAlterDatabaseStmt)); - case QUERY_NODE_FLUSH_DATABASE_STMT: - return makeNode(type, sizeof(SFlushDatabaseStmt)); - case QUERY_NODE_TRIM_DATABASE_STMT: - return makeNode(type, sizeof(STrimDatabaseStmt)); - case QUERY_NODE_CREATE_TABLE_STMT: - return makeNode(type, sizeof(SCreateTableStmt)); - case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: - return makeNode(type, sizeof(SCreateSubTableClause)); - case QUERY_NODE_CREATE_MULTI_TABLES_STMT: - return makeNode(type, sizeof(SCreateMultiTablesStmt)); - case QUERY_NODE_DROP_TABLE_CLAUSE: - return makeNode(type, sizeof(SDropTableClause)); - case QUERY_NODE_DROP_TABLE_STMT: - return makeNode(type, sizeof(SDropTableStmt)); - case QUERY_NODE_DROP_SUPER_TABLE_STMT: - return makeNode(type, sizeof(SDropSuperTableStmt)); - case QUERY_NODE_ALTER_TABLE_STMT: - case QUERY_NODE_ALTER_SUPER_TABLE_STMT: - return makeNode(type, sizeof(SAlterTableStmt)); - case QUERY_NODE_CREATE_USER_STMT: - return makeNode(type, sizeof(SCreateUserStmt)); - case QUERY_NODE_ALTER_USER_STMT: - return makeNode(type, sizeof(SAlterUserStmt)); - case QUERY_NODE_DROP_USER_STMT: - return makeNode(type, sizeof(SDropUserStmt)); - case QUERY_NODE_USE_DATABASE_STMT: - return makeNode(type, sizeof(SUseDatabaseStmt)); - case QUERY_NODE_CREATE_DNODE_STMT: - return makeNode(type, sizeof(SCreateDnodeStmt)); - case QUERY_NODE_DROP_DNODE_STMT: - return makeNode(type, sizeof(SDropDnodeStmt)); - case QUERY_NODE_ALTER_DNODE_STMT: - return makeNode(type, sizeof(SAlterDnodeStmt)); - case QUERY_NODE_CREATE_INDEX_STMT: - return makeNode(type, sizeof(SCreateIndexStmt)); - case QUERY_NODE_DROP_INDEX_STMT: - return makeNode(type, sizeof(SDropIndexStmt)); - case QUERY_NODE_CREATE_QNODE_STMT: - case QUERY_NODE_CREATE_BNODE_STMT: - case QUERY_NODE_CREATE_SNODE_STMT: - case QUERY_NODE_CREATE_MNODE_STMT: - return makeNode(type, sizeof(SCreateComponentNodeStmt)); - case QUERY_NODE_DROP_QNODE_STMT: - case QUERY_NODE_DROP_BNODE_STMT: - case QUERY_NODE_DROP_SNODE_STMT: - case QUERY_NODE_DROP_MNODE_STMT: - return makeNode(type, sizeof(SDropComponentNodeStmt)); - case QUERY_NODE_CREATE_TOPIC_STMT: - return makeNode(type, sizeof(SCreateTopicStmt)); - case QUERY_NODE_DROP_TOPIC_STMT: - return makeNode(type, sizeof(SDropTopicStmt)); - case QUERY_NODE_DROP_CGROUP_STMT: - return makeNode(type, sizeof(SDropCGroupStmt)); - case QUERY_NODE_ALTER_LOCAL_STMT: - return makeNode(type, sizeof(SAlterLocalStmt)); - case QUERY_NODE_EXPLAIN_STMT: - return makeNode(type, sizeof(SExplainStmt)); - case QUERY_NODE_DESCRIBE_STMT: - return makeNode(type, sizeof(SDescribeStmt)); - case QUERY_NODE_RESET_QUERY_CACHE_STMT: - return makeNode(type, sizeof(SNode)); - case QUERY_NODE_COMPACT_DATABASE_STMT: - return makeNode(type, sizeof(SCompactDatabaseStmt)); - case QUERY_NODE_CREATE_FUNCTION_STMT: - return makeNode(type, sizeof(SCreateFunctionStmt)); - case QUERY_NODE_DROP_FUNCTION_STMT: - return makeNode(type, sizeof(SDropFunctionStmt)); - case QUERY_NODE_CREATE_STREAM_STMT: - return makeNode(type, sizeof(SCreateStreamStmt)); - case QUERY_NODE_DROP_STREAM_STMT: - return makeNode(type, sizeof(SDropStreamStmt)); - case QUERY_NODE_PAUSE_STREAM_STMT: - return makeNode(type, sizeof(SPauseStreamStmt)); - case QUERY_NODE_RESUME_STREAM_STMT: - return makeNode(type, sizeof(SResumeStreamStmt)); - case QUERY_NODE_BALANCE_VGROUP_STMT: - return makeNode(type, sizeof(SBalanceVgroupStmt)); - case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: - return makeNode(type, sizeof(SBalanceVgroupLeaderStmt)); - case QUERY_NODE_MERGE_VGROUP_STMT: - return makeNode(type, sizeof(SMergeVgroupStmt)); - case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: - return makeNode(type, sizeof(SRedistributeVgroupStmt)); - case QUERY_NODE_SPLIT_VGROUP_STMT: - return makeNode(type, sizeof(SSplitVgroupStmt)); - case QUERY_NODE_SYNCDB_STMT: - break; - case QUERY_NODE_GRANT_STMT: - return makeNode(type, sizeof(SGrantStmt)); - case QUERY_NODE_REVOKE_STMT: - return makeNode(type, sizeof(SRevokeStmt)); - case QUERY_NODE_SHOW_DNODES_STMT: - case QUERY_NODE_SHOW_MNODES_STMT: - case QUERY_NODE_SHOW_MODULES_STMT: - case QUERY_NODE_SHOW_QNODES_STMT: - case QUERY_NODE_SHOW_SNODES_STMT: - case QUERY_NODE_SHOW_BNODES_STMT: - case QUERY_NODE_SHOW_CLUSTER_STMT: - case QUERY_NODE_SHOW_DATABASES_STMT: - case QUERY_NODE_SHOW_FUNCTIONS_STMT: - case QUERY_NODE_SHOW_INDEXES_STMT: - case QUERY_NODE_SHOW_STABLES_STMT: - case QUERY_NODE_SHOW_STREAMS_STMT: - case QUERY_NODE_SHOW_TABLES_STMT: - case QUERY_NODE_SHOW_USERS_STMT: - case QUERY_NODE_SHOW_LICENCES_STMT: - case QUERY_NODE_SHOW_VGROUPS_STMT: - case QUERY_NODE_SHOW_TOPICS_STMT: - case QUERY_NODE_SHOW_CONSUMERS_STMT: - case QUERY_NODE_SHOW_CONNECTIONS_STMT: - case QUERY_NODE_SHOW_QUERIES_STMT: - case QUERY_NODE_SHOW_VNODES_STMT: - case QUERY_NODE_SHOW_APPS_STMT: - case QUERY_NODE_SHOW_SCORES_STMT: - case QUERY_NODE_SHOW_VARIABLES_STMT: - case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: - case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: - case QUERY_NODE_SHOW_TAGS_STMT: - case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: - case QUERY_NODE_SHOW_VIEWS_STMT: - return makeNode(type, sizeof(SShowStmt)); - case QUERY_NODE_SHOW_TABLE_TAGS_STMT: - return makeNode(type, sizeof(SShowTableTagsStmt)); - case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: - return makeNode(type, sizeof(SShowDnodeVariablesStmt)); - case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: - return makeNode(type, sizeof(SShowCreateDatabaseStmt)); - case QUERY_NODE_SHOW_DB_ALIVE_STMT: - case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: - return makeNode(type, sizeof(SShowAliveStmt)); - case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - return makeNode(type, sizeof(SShowCreateTableStmt)); - case QUERY_NODE_SHOW_CREATE_VIEW_STMT: - return makeNode(type, sizeof(SShowCreateViewStmt)); - case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: - return makeNode(type, sizeof(SShowTableDistributedStmt)); - case QUERY_NODE_KILL_QUERY_STMT: - return makeNode(type, sizeof(SKillQueryStmt)); - case QUERY_NODE_KILL_TRANSACTION_STMT: - case QUERY_NODE_KILL_CONNECTION_STMT: - return makeNode(type, sizeof(SKillStmt)); - case QUERY_NODE_DELETE_STMT: - return makeNode(type, sizeof(SDeleteStmt)); - case QUERY_NODE_INSERT_STMT: - return makeNode(type, sizeof(SInsertStmt)); - case QUERY_NODE_QUERY: - return makeNode(type, sizeof(SQuery)); - case QUERY_NODE_RESTORE_DNODE_STMT: - case QUERY_NODE_RESTORE_QNODE_STMT: - case QUERY_NODE_RESTORE_MNODE_STMT: - case QUERY_NODE_RESTORE_VNODE_STMT: - return makeNode(type, sizeof(SRestoreComponentNodeStmt)); - case QUERY_NODE_CREATE_VIEW_STMT: - return makeNode(type, sizeof(SCreateViewStmt)); - case QUERY_NODE_DROP_VIEW_STMT: - return makeNode(type, sizeof(SDropViewStmt)); - case QUERY_NODE_LOGIC_PLAN_SCAN: - return makeNode(type, sizeof(SScanLogicNode)); - case QUERY_NODE_LOGIC_PLAN_JOIN: - return makeNode(type, sizeof(SJoinLogicNode)); - case QUERY_NODE_LOGIC_PLAN_AGG: - return makeNode(type, sizeof(SAggLogicNode)); - case QUERY_NODE_LOGIC_PLAN_PROJECT: - return makeNode(type, sizeof(SProjectLogicNode)); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: - return makeNode(type, sizeof(SVnodeModifyLogicNode)); - case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - return makeNode(type, sizeof(SExchangeLogicNode)); - case QUERY_NODE_LOGIC_PLAN_MERGE: - return makeNode(type, sizeof(SMergeLogicNode)); - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return makeNode(type, sizeof(SWindowLogicNode)); - case QUERY_NODE_LOGIC_PLAN_FILL: - return makeNode(type, sizeof(SFillLogicNode)); - case QUERY_NODE_LOGIC_PLAN_SORT: - return makeNode(type, sizeof(SSortLogicNode)); - case QUERY_NODE_LOGIC_PLAN_PARTITION: - return makeNode(type, sizeof(SPartitionLogicNode)); - case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: - return makeNode(type, sizeof(SIndefRowsFuncLogicNode)); - case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: - return makeNode(type, sizeof(SInterpFuncLogicNode)); - case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: - return makeNode(type, sizeof(SGroupCacheLogicNode)); - case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL: - return makeNode(type, sizeof(SDynQueryCtrlLogicNode)); - case QUERY_NODE_LOGIC_SUBPLAN: - return makeNode(type, sizeof(SLogicSubplan)); - case QUERY_NODE_LOGIC_PLAN: - return makeNode(type, sizeof(SQueryLogicPlan)); - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - return makeNode(type, sizeof(STagScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - return makeNode(type, sizeof(STableScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - return makeNode(type, sizeof(STableSeqScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - return makeNode(type, sizeof(STableMergeScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return makeNode(type, sizeof(SStreamScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - return makeNode(type, sizeof(SSystemTableScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - return makeNode(type, sizeof(SBlockDistScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - return makeNode(type, sizeof(SLastRowScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: - return makeNode(type, sizeof(STableCountScanPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - return makeNode(type, sizeof(SProjectPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return makeNode(type, sizeof(SSortMergeJoinPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: - return makeNode(type, sizeof(SHashJoinPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: - return makeNode(type, sizeof(SAggPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - return makeNode(type, sizeof(SExchangePhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE: - return makeNode(type, sizeof(SMergePhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_SORT: - return makeNode(type, sizeof(SSortPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: - return makeNode(type, sizeof(SGroupSortPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - return makeNode(type, sizeof(SIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: - return makeNode(type, sizeof(SMergeAlignedIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - return makeNode(type, sizeof(SStreamIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: - return makeNode(type, sizeof(SStreamFinalIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_FILL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: - return makeNode(type, sizeof(SFillPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: - return makeNode(type, sizeof(SSessionWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: - return makeNode(type, sizeof(SStreamSessionWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: - return makeNode(type, sizeof(SStreamSemiSessionWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - return makeNode(type, sizeof(SStreamFinalSessionWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: - return makeNode(type, sizeof(SStateWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: - return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: - return makeNode(type, sizeof(SEventWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: - return makeNode(type, sizeof(SStreamEventWinodwPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: - return makeNode(type, sizeof(SPartitionPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: - return makeNode(type, sizeof(SStreamPartitionPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: - return makeNode(type, sizeof(SIndefRowsFuncPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: - return makeNode(type, sizeof(SInterpFuncLogicNode)); - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - return makeNode(type, sizeof(SDataDispatcherNode)); - case QUERY_NODE_PHYSICAL_PLAN_INSERT: - return makeNode(type, sizeof(SDataInserterNode)); - case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: - return makeNode(type, sizeof(SQueryInserterNode)); - case QUERY_NODE_PHYSICAL_PLAN_DELETE: - return makeNode(type, sizeof(SDataDeleterNode)); - case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: - return makeNode(type, sizeof(SGroupCachePhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: - return makeNode(type, sizeof(SDynQueryCtrlPhysiNode)); - case QUERY_NODE_PHYSICAL_SUBPLAN: - return makeNode(type, sizeof(SSubplan)); - case QUERY_NODE_PHYSICAL_PLAN: - return makeNode(type, sizeof(SQueryPlan)); - default: - break; + int32_t size = getNodeSize(type); + if (size > 0) { + return makeNode(type, size); } - nodesError("nodesMakeNode unknown node = %s", nodesNodeName(type)); + nodesError("nodesMakeNode unsupported type = %d", type); return NULL; } -static void destroyVgDataBlockArray(SArray* pArray) { - size_t size = taosArrayGetSize(pArray); - for (size_t i = 0; i < size; ++i) { - SVgDataBlocks* pVg = taosArrayGetP(pArray, i); - taosMemoryFreeClear(pVg->pData); - taosMemoryFreeClear(pVg); - } - taosArrayDestroy(pArray); -} - -static void destroyLogicNode(SLogicNode* pNode) { - nodesDestroyList(pNode->pTargets); - nodesDestroyNode(pNode->pConditions); - nodesDestroyList(pNode->pChildren); - nodesDestroyNode(pNode->pLimit); - nodesDestroyNode(pNode->pSlimit); - nodesDestroyList(pNode->pHint); -} - -static void destroyPhysiNode(SPhysiNode* pNode) { - nodesDestroyList(pNode->pChildren); - nodesDestroyNode(pNode->pConditions); - nodesDestroyNode((SNode*)pNode->pOutputDataBlockDesc); - nodesDestroyNode(pNode->pLimit); - nodesDestroyNode(pNode->pSlimit); -} - -static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { - destroyPhysiNode((SPhysiNode*)pNode); - nodesDestroyList(pNode->pExprs); - nodesDestroyList(pNode->pFuncs); - nodesDestroyNode(pNode->pTspk); - nodesDestroyNode(pNode->pTsEnd); -} - -static void destroyPartitionPhysiNode(SPartitionPhysiNode* pNode) { - destroyPhysiNode((SPhysiNode*)pNode); - nodesDestroyList(pNode->pExprs); - nodesDestroyList(pNode->pPartitionKeys); - nodesDestroyList(pNode->pTargets); -} - -static void destroyScanPhysiNode(SScanPhysiNode* pNode) { - destroyPhysiNode((SPhysiNode*)pNode); - nodesDestroyList(pNode->pScanCols); - nodesDestroyList(pNode->pScanPseudoCols); -} - -static void destroyDataSinkNode(SDataSinkNode* pNode) { nodesDestroyNode((SNode*)pNode->pInputDataBlockDesc); } - -static void destroyExprNode(SExprNode* pExpr) { taosArrayDestroy(pExpr->pAssociation); } - -static void destroyTableCfg(STableCfg* pCfg) { - if (NULL == pCfg) { - return; - } - taosArrayDestroy(pCfg->pFuncs); - taosMemoryFree(pCfg->pComment); - taosMemoryFree(pCfg->pSchemas); - taosMemoryFree(pCfg->pTags); - taosMemoryFree(pCfg); -} - -static void destroySmaIndex(void* pIndex) { taosMemoryFree(((STableIndexInfo*)pIndex)->expr); } - -static void destroyHintValue(EHintOption option, void* value) { - switch (option) { - default: - break; - } - - taosMemoryFree(value); -} - -void nodesDestroyNode(SNode* pNode) { - if (NULL == pNode) { - return; - } - - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: - destroyExprNode((SExprNode*)pNode); - break; - case QUERY_NODE_VALUE: { - SValueNode* pValue = (SValueNode*)pNode; - destroyExprNode((SExprNode*)pNode); - taosMemoryFreeClear(pValue->literal); - if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { - taosMemoryFreeClear(pValue->datum.p); - } - break; - } - case QUERY_NODE_OPERATOR: { - SOperatorNode* pOp = (SOperatorNode*)pNode; - destroyExprNode((SExprNode*)pNode); - nodesDestroyNode(pOp->pLeft); - nodesDestroyNode(pOp->pRight); - break; - } - case QUERY_NODE_LOGIC_CONDITION: - destroyExprNode((SExprNode*)pNode); - nodesDestroyList(((SLogicConditionNode*)pNode)->pParameterList); - break; - case QUERY_NODE_FUNCTION: - destroyExprNode((SExprNode*)pNode); - nodesDestroyList(((SFunctionNode*)pNode)->pParameterList); - break; - case QUERY_NODE_REAL_TABLE: { - SRealTableNode* pReal = (SRealTableNode*)pNode; - taosMemoryFreeClear(pReal->pMeta); - taosMemoryFreeClear(pReal->pVgroupList); - taosArrayDestroyEx(pReal->pSmaIndexes, destroySmaIndex); - break; - } - case QUERY_NODE_TEMP_TABLE: - nodesDestroyNode(((STempTableNode*)pNode)->pSubquery); - break; - case QUERY_NODE_JOIN_TABLE: { - SJoinTableNode* pJoin = (SJoinTableNode*)pNode; - nodesDestroyNode(pJoin->pLeft); - nodesDestroyNode(pJoin->pRight); - nodesDestroyNode(pJoin->pOnCond); - break; - } - case QUERY_NODE_GROUPING_SET: - nodesDestroyList(((SGroupingSetNode*)pNode)->pParameterList); - break; - case QUERY_NODE_ORDER_BY_EXPR: - nodesDestroyNode(((SOrderByExprNode*)pNode)->pExpr); - break; - case QUERY_NODE_LIMIT: // no pointer field - break; - case QUERY_NODE_STATE_WINDOW: { - SStateWindowNode* pState = (SStateWindowNode*)pNode; - nodesDestroyNode(pState->pCol); - nodesDestroyNode(pState->pExpr); - break; - } - case QUERY_NODE_SESSION_WINDOW: { - SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; - nodesDestroyNode((SNode*)pSession->pCol); - nodesDestroyNode((SNode*)pSession->pGap); - break; - } - case QUERY_NODE_INTERVAL_WINDOW: { - SIntervalWindowNode* pJoin = (SIntervalWindowNode*)pNode; - nodesDestroyNode(pJoin->pCol); - nodesDestroyNode(pJoin->pInterval); - nodesDestroyNode(pJoin->pOffset); - nodesDestroyNode(pJoin->pSliding); - nodesDestroyNode(pJoin->pFill); - break; - } - case QUERY_NODE_NODE_LIST: - nodesDestroyList(((SNodeListNode*)pNode)->pNodeList); - break; - case QUERY_NODE_FILL: { - SFillNode* pFill = (SFillNode*)pNode; - nodesDestroyNode(pFill->pValues); - nodesDestroyNode(pFill->pWStartTs); - break; - } - case QUERY_NODE_RAW_EXPR: - nodesDestroyNode(((SRawExprNode*)pNode)->pNode); - break; - case QUERY_NODE_TARGET: - nodesDestroyNode(((STargetNode*)pNode)->pExpr); - break; - case QUERY_NODE_DATABLOCK_DESC: - nodesDestroyList(((SDataBlockDescNode*)pNode)->pSlots); - break; - case QUERY_NODE_SLOT_DESC: // no pointer field - case QUERY_NODE_COLUMN_DEF: // no pointer field - case QUERY_NODE_DOWNSTREAM_SOURCE: // no pointer field - break; - case QUERY_NODE_DATABASE_OPTIONS: { - SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode; - nodesDestroyNode((SNode*)pOptions->pDaysPerFile); - nodesDestroyList(pOptions->pKeep); - nodesDestroyList(pOptions->pRetentions); - break; - } - case QUERY_NODE_TABLE_OPTIONS: { - STableOptions* pOptions = (STableOptions*)pNode; - nodesDestroyList(pOptions->pMaxDelay); - nodesDestroyList(pOptions->pWatermark); - nodesDestroyList(pOptions->pRollupFuncs); - nodesDestroyList(pOptions->pSma); - nodesDestroyList(pOptions->pDeleteMark); - break; - } - case QUERY_NODE_INDEX_OPTIONS: { - SIndexOptions* pOptions = (SIndexOptions*)pNode; - nodesDestroyList(pOptions->pFuncs); - nodesDestroyNode(pOptions->pInterval); - nodesDestroyNode(pOptions->pOffset); - nodesDestroyNode(pOptions->pSliding); - nodesDestroyNode(pOptions->pStreamOptions); - break; - } - case QUERY_NODE_EXPLAIN_OPTIONS: // no pointer field - break; - case QUERY_NODE_STREAM_OPTIONS: { - SStreamOptions* pOptions = (SStreamOptions*)pNode; - nodesDestroyNode(pOptions->pDelay); - nodesDestroyNode(pOptions->pWatermark); - nodesDestroyNode(pOptions->pDeleteMark); - break; - } - case QUERY_NODE_LEFT_VALUE: // no pointer field - case QUERY_NODE_COLUMN_REF: // no pointer field - break; - case QUERY_NODE_WHEN_THEN: { - SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; - nodesDestroyNode(pWhenThen->pWhen); - nodesDestroyNode(pWhenThen->pThen); - break; - } - case QUERY_NODE_CASE_WHEN: { - SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)pNode; - nodesDestroyNode(pCaseWhen->pCase); - nodesDestroyNode(pCaseWhen->pElse); - nodesDestroyList(pCaseWhen->pWhenThenList); - break; - } - case QUERY_NODE_EVENT_WINDOW: { - SEventWindowNode* pEvent = (SEventWindowNode*)pNode; - nodesDestroyNode(pEvent->pCol); - nodesDestroyNode(pEvent->pStartCond); - nodesDestroyNode(pEvent->pEndCond); - break; - } - case QUERY_NODE_HINT: { - SHintNode* pHint = (SHintNode*)pNode; - destroyHintValue(pHint->option, pHint->value); - break; - } - case QUERY_NODE_VIEW: { - SViewNode* pView = (SViewNode*)pNode; - taosMemoryFreeClear(pView->pMeta); - taosMemoryFreeClear(pView->pVgroupList); - taosArrayDestroyEx(pView->pSmaIndexes, destroySmaIndex); - break; - } - case QUERY_NODE_SET_OPERATOR: { - SSetOperator* pStmt = (SSetOperator*)pNode; - nodesDestroyList(pStmt->pProjectionList); - nodesDestroyNode(pStmt->pLeft); - nodesDestroyNode(pStmt->pRight); - nodesDestroyList(pStmt->pOrderByList); - nodesDestroyNode(pStmt->pLimit); - break; - } - case QUERY_NODE_SELECT_STMT: { - SSelectStmt* pStmt = (SSelectStmt*)pNode; - nodesDestroyList(pStmt->pProjectionList); - nodesDestroyNode(pStmt->pFromTable); - nodesDestroyNode(pStmt->pWhere); - nodesDestroyList(pStmt->pPartitionByList); - nodesDestroyList(pStmt->pTags); - nodesDestroyNode(pStmt->pSubtable); - nodesDestroyNode(pStmt->pWindow); - nodesDestroyList(pStmt->pGroupByList); - nodesDestroyNode(pStmt->pHaving); - nodesDestroyNode(pStmt->pRange); - nodesDestroyNode(pStmt->pEvery); - nodesDestroyNode(pStmt->pFill); - nodesDestroyList(pStmt->pOrderByList); - nodesDestroyNode((SNode*)pStmt->pLimit); - nodesDestroyNode((SNode*)pStmt->pSlimit); - nodesDestroyList(pStmt->pHint); - break; - } - case QUERY_NODE_VNODE_MODIFY_STMT: { - SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode; - destroyVgDataBlockArray(pStmt->pDataBlocks); - taosMemoryFreeClear(pStmt->pTableMeta); - nodesDestroyNode(pStmt->pTagCond); - taosArrayDestroy(pStmt->pTableTag); - taosHashCleanup(pStmt->pVgroupsHashObj); - taosHashCleanup(pStmt->pSubTableHashObj); - taosHashCleanup(pStmt->pTableNameHashObj); - taosHashCleanup(pStmt->pDbFNameHashObj); - if (pStmt->freeHashFunc) { - pStmt->freeHashFunc(pStmt->pTableBlockHashObj); - } - if (pStmt->freeArrayFunc) { - pStmt->freeArrayFunc(pStmt->pVgDataBlocks); - } - tdDestroySVCreateTbReq(pStmt->pCreateTblReq); - taosMemoryFreeClear(pStmt->pCreateTblReq); - if (pStmt->freeStbRowsCxtFunc) { - pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt); - } - taosMemoryFreeClear(pStmt->pStbRowsCxt); - taosCloseFile(&pStmt->fp); - break; - } - case QUERY_NODE_CREATE_DATABASE_STMT: - nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions); - break; - case QUERY_NODE_DROP_DATABASE_STMT: // no pointer field - break; - case QUERY_NODE_ALTER_DATABASE_STMT: - nodesDestroyNode((SNode*)((SAlterDatabaseStmt*)pNode)->pOptions); - break; - case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field - case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field - break; - case QUERY_NODE_CREATE_TABLE_STMT: { - SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode; - nodesDestroyList(pStmt->pCols); - nodesDestroyList(pStmt->pTags); - nodesDestroyNode((SNode*)pStmt->pOptions); - break; - } - case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: { - SCreateSubTableClause* pStmt = (SCreateSubTableClause*)pNode; - nodesDestroyList(pStmt->pSpecificTags); - nodesDestroyList(pStmt->pValsOfTags); - nodesDestroyNode((SNode*)pStmt->pOptions); - break; - } - case QUERY_NODE_CREATE_MULTI_TABLES_STMT: - nodesDestroyList(((SCreateMultiTablesStmt*)pNode)->pSubTables); - break; - case QUERY_NODE_DROP_TABLE_CLAUSE: // no pointer field - break; - case QUERY_NODE_DROP_TABLE_STMT: - nodesDestroyList(((SDropTableStmt*)pNode)->pTables); - break; - case QUERY_NODE_DROP_SUPER_TABLE_STMT: // no pointer field - break; - case QUERY_NODE_ALTER_TABLE_STMT: - case QUERY_NODE_ALTER_SUPER_TABLE_STMT: { - SAlterTableStmt* pStmt = (SAlterTableStmt*)pNode; - nodesDestroyNode((SNode*)pStmt->pOptions); - nodesDestroyNode((SNode*)pStmt->pVal); - break; - } - case QUERY_NODE_CREATE_USER_STMT: { - SCreateUserStmt* pStmt = (SCreateUserStmt*)pNode; - taosMemoryFree(pStmt->pIpRanges); - nodesDestroyList(pStmt->pNodeListIpRanges); - break; - } - case QUERY_NODE_ALTER_USER_STMT: { - SAlterUserStmt* pStmt = (SAlterUserStmt*)pNode; - taosMemoryFree(pStmt->pIpRanges); - nodesDestroyList(pStmt->pNodeListIpRanges); - } - case QUERY_NODE_DROP_USER_STMT: // no pointer field - case QUERY_NODE_USE_DATABASE_STMT: // no pointer field - case QUERY_NODE_CREATE_DNODE_STMT: // no pointer field - case QUERY_NODE_DROP_DNODE_STMT: // no pointer field - case QUERY_NODE_ALTER_DNODE_STMT: // no pointer field - break; - case QUERY_NODE_CREATE_INDEX_STMT: { - SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode; - nodesDestroyNode((SNode*)pStmt->pOptions); - nodesDestroyList(pStmt->pCols); - if (pStmt->pReq) { - tFreeSMCreateSmaReq(pStmt->pReq); - taosMemoryFreeClear(pStmt->pReq); - } - break; - } - case QUERY_NODE_DROP_INDEX_STMT: // no pointer field - case QUERY_NODE_CREATE_QNODE_STMT: // no pointer field - case QUERY_NODE_DROP_QNODE_STMT: // no pointer field - case QUERY_NODE_CREATE_BNODE_STMT: // no pointer field - case QUERY_NODE_DROP_BNODE_STMT: // no pointer field - case QUERY_NODE_CREATE_SNODE_STMT: // no pointer field - case QUERY_NODE_DROP_SNODE_STMT: // no pointer field - case QUERY_NODE_CREATE_MNODE_STMT: // no pointer field - case QUERY_NODE_DROP_MNODE_STMT: // no pointer field - break; - case QUERY_NODE_CREATE_TOPIC_STMT: - nodesDestroyNode(((SCreateTopicStmt*)pNode)->pQuery); - nodesDestroyNode(((SCreateTopicStmt*)pNode)->pWhere); - break; - case QUERY_NODE_DROP_TOPIC_STMT: // no pointer field - case QUERY_NODE_DROP_CGROUP_STMT: // no pointer field - case QUERY_NODE_ALTER_LOCAL_STMT: // no pointer field - break; - case QUERY_NODE_EXPLAIN_STMT: { - SExplainStmt* pStmt = (SExplainStmt*)pNode; - nodesDestroyNode((SNode*)pStmt->pOptions); - nodesDestroyNode(pStmt->pQuery); - break; - } - case QUERY_NODE_DESCRIBE_STMT: - taosMemoryFree(((SDescribeStmt*)pNode)->pMeta); - break; - case QUERY_NODE_RESET_QUERY_CACHE_STMT: // no pointer field - break; - case QUERY_NODE_COMPACT_DATABASE_STMT: { - SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)pNode; - nodesDestroyNode(pStmt->pStart); - nodesDestroyNode(pStmt->pEnd); - break; - } - case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field - case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field - break; - case QUERY_NODE_CREATE_STREAM_STMT: { - SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pNode; - nodesDestroyNode((SNode*)pStmt->pOptions); - nodesDestroyNode(pStmt->pQuery); - nodesDestroyList(pStmt->pTags); - nodesDestroyNode(pStmt->pSubtable); - tFreeSCMCreateStreamReq(pStmt->pReq); - taosMemoryFreeClear(pStmt->pReq); - break; - } - case QUERY_NODE_DROP_STREAM_STMT: // no pointer field - case QUERY_NODE_PAUSE_STREAM_STMT: // no pointer field - case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field - case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field - case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field - case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field - break; - case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: - nodesDestroyList(((SRedistributeVgroupStmt*)pNode)->pDnodes); - break; - case QUERY_NODE_SPLIT_VGROUP_STMT: // no pointer field - case QUERY_NODE_SYNCDB_STMT: // no pointer field - break; - case QUERY_NODE_GRANT_STMT: - nodesDestroyNode(((SGrantStmt*)pNode)->pTagCond); - break; - case QUERY_NODE_REVOKE_STMT: - nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond); - break; - case QUERY_NODE_SHOW_DNODES_STMT: - case QUERY_NODE_SHOW_MNODES_STMT: - case QUERY_NODE_SHOW_MODULES_STMT: - case QUERY_NODE_SHOW_QNODES_STMT: - case QUERY_NODE_SHOW_SNODES_STMT: - case QUERY_NODE_SHOW_BNODES_STMT: - case QUERY_NODE_SHOW_CLUSTER_STMT: - case QUERY_NODE_SHOW_DATABASES_STMT: - case QUERY_NODE_SHOW_FUNCTIONS_STMT: - case QUERY_NODE_SHOW_INDEXES_STMT: - case QUERY_NODE_SHOW_STABLES_STMT: - case QUERY_NODE_SHOW_STREAMS_STMT: - case QUERY_NODE_SHOW_TABLES_STMT: - case QUERY_NODE_SHOW_USERS_STMT: - case QUERY_NODE_SHOW_LICENCES_STMT: - case QUERY_NODE_SHOW_VGROUPS_STMT: - case QUERY_NODE_SHOW_TOPICS_STMT: - case QUERY_NODE_SHOW_CONSUMERS_STMT: - case QUERY_NODE_SHOW_CONNECTIONS_STMT: - case QUERY_NODE_SHOW_QUERIES_STMT: - case QUERY_NODE_SHOW_VNODES_STMT: - case QUERY_NODE_SHOW_APPS_STMT: - case QUERY_NODE_SHOW_SCORES_STMT: - case QUERY_NODE_SHOW_VARIABLES_STMT: - case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: - case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: - case QUERY_NODE_SHOW_TAGS_STMT: - case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: - case QUERY_NODE_SHOW_VIEWS_STMT: { - SShowStmt* pStmt = (SShowStmt*)pNode; - nodesDestroyNode(pStmt->pDbName); - nodesDestroyNode(pStmt->pTbName); - break; - } - case QUERY_NODE_SHOW_TABLE_TAGS_STMT: { - SShowTableTagsStmt* pStmt = (SShowTableTagsStmt*)pNode; - nodesDestroyNode(pStmt->pDbName); - nodesDestroyNode(pStmt->pTbName); - nodesDestroyList(pStmt->pTags); - break; - } - case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: - nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pDnodeId); - nodesDestroyNode(((SShowDnodeVariablesStmt*)pNode)->pLikePattern); - break; - case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: - taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg); - break; - case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - taosMemoryFreeClear(((SShowCreateTableStmt*)pNode)->pDbCfg); - destroyTableCfg((STableCfg*)(((SShowCreateTableStmt*)pNode)->pTableCfg)); - break; - case QUERY_NODE_SHOW_CREATE_VIEW_STMT: // no pointer field - case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: // no pointer field - case QUERY_NODE_KILL_CONNECTION_STMT: // no pointer field - case QUERY_NODE_KILL_QUERY_STMT: // no pointer field - case QUERY_NODE_KILL_TRANSACTION_STMT: // no pointer field - break; - case QUERY_NODE_DELETE_STMT: { - SDeleteStmt* pStmt = (SDeleteStmt*)pNode; - nodesDestroyNode(pStmt->pFromTable); - nodesDestroyNode(pStmt->pWhere); - nodesDestroyNode(pStmt->pCountFunc); - nodesDestroyNode(pStmt->pFirstFunc); - nodesDestroyNode(pStmt->pLastFunc); - nodesDestroyNode(pStmt->pTagCond); - break; - } - case QUERY_NODE_INSERT_STMT: { - SInsertStmt* pStmt = (SInsertStmt*)pNode; - nodesDestroyNode(pStmt->pTable); - nodesDestroyList(pStmt->pCols); - nodesDestroyNode(pStmt->pQuery); - break; - } - case QUERY_NODE_QUERY: { - SQuery* pQuery = (SQuery*)pNode; - nodesDestroyNode(pQuery->pPrevRoot); - nodesDestroyNode(pQuery->pRoot); - nodesDestroyNode(pQuery->pPostRoot); - taosMemoryFreeClear(pQuery->pResSchema); - if (NULL != pQuery->pCmdMsg) { - taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); - taosMemoryFreeClear(pQuery->pCmdMsg); - } - taosArrayDestroy(pQuery->pDbList); - taosArrayDestroy(pQuery->pTableList); - taosArrayDestroy(pQuery->pTargetTableList); - taosArrayDestroy(pQuery->pPlaceholderValues); - nodesDestroyNode(pQuery->pPrepareRoot); - break; - } - case QUERY_NODE_RESTORE_DNODE_STMT: // no pointer field - case QUERY_NODE_RESTORE_QNODE_STMT: // no pointer field - case QUERY_NODE_RESTORE_MNODE_STMT: // no pointer field - case QUERY_NODE_RESTORE_VNODE_STMT: // no pointer field - break; - case QUERY_NODE_CREATE_VIEW_STMT: { - SCreateViewStmt* pStmt = (SCreateViewStmt*)pNode; - taosMemoryFree(pStmt->pQuerySql); - tFreeSCMCreateViewReq(&pStmt->createReq); - nodesDestroyNode(pStmt->pQuery); - break; - } - case QUERY_NODE_DROP_VIEW_STMT: - break; - case QUERY_NODE_LOGIC_PLAN_SCAN: { - SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pScanCols); - nodesDestroyList(pLogicNode->pScanPseudoCols); - taosMemoryFreeClear(pLogicNode->pVgroupList); - nodesDestroyList(pLogicNode->pDynamicScanFuncs); - nodesDestroyNode(pLogicNode->pTagCond); - nodesDestroyNode(pLogicNode->pTagIndexCond); - taosArrayDestroyEx(pLogicNode->pSmaIndexes, destroySmaIndex); - nodesDestroyList(pLogicNode->pGroupTags); - nodesDestroyList(pLogicNode->pTags); - nodesDestroyNode(pLogicNode->pSubtable); - break; - } - case QUERY_NODE_LOGIC_PLAN_JOIN: { - SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyNode(pLogicNode->pPrimKeyEqCond); - nodesDestroyNode(pLogicNode->pOtherOnCond); - nodesDestroyNode(pLogicNode->pColEqCond); - break; - } - case QUERY_NODE_LOGIC_PLAN_AGG: { - SAggLogicNode* pLogicNode = (SAggLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pAggFuncs); - nodesDestroyList(pLogicNode->pGroupKeys); - break; - } - case QUERY_NODE_LOGIC_PLAN_PROJECT: { - SProjectLogicNode* pLogicNode = (SProjectLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pProjections); - break; - } - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: { - SVnodeModifyLogicNode* pLogicNode = (SVnodeModifyLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - destroyVgDataBlockArray(pLogicNode->pDataBlocks); - // pVgDataBlocks is weak reference - nodesDestroyNode(pLogicNode->pAffectedRows); - nodesDestroyNode(pLogicNode->pStartTs); - nodesDestroyNode(pLogicNode->pEndTs); - taosMemoryFreeClear(pLogicNode->pVgroupList); - nodesDestroyList(pLogicNode->pInsertCols); - break; - } - case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - destroyLogicNode((SLogicNode*)pNode); - break; - case QUERY_NODE_LOGIC_PLAN_MERGE: { - SMergeLogicNode* pLogicNode = (SMergeLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pMergeKeys); - nodesDestroyList(pLogicNode->pInputs); - break; - } - case QUERY_NODE_LOGIC_PLAN_WINDOW: { - SWindowLogicNode* pLogicNode = (SWindowLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pFuncs); - nodesDestroyNode(pLogicNode->pTspk); - nodesDestroyNode(pLogicNode->pTsEnd); - nodesDestroyNode(pLogicNode->pStateExpr); - nodesDestroyNode(pLogicNode->pStartCond); - nodesDestroyNode(pLogicNode->pEndCond); - break; - } - case QUERY_NODE_LOGIC_PLAN_FILL: { - SFillLogicNode* pLogicNode = (SFillLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyNode(pLogicNode->pWStartTs); - nodesDestroyNode(pLogicNode->pValues); - nodesDestroyList(pLogicNode->pFillExprs); - nodesDestroyList(pLogicNode->pNotFillExprs); - break; - } - case QUERY_NODE_LOGIC_PLAN_SORT: { - SSortLogicNode* pLogicNode = (SSortLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pSortKeys); - break; - } - case QUERY_NODE_LOGIC_PLAN_PARTITION: { - SPartitionLogicNode* pLogicNode = (SPartitionLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pPartitionKeys); - nodesDestroyList(pLogicNode->pTags); - nodesDestroyNode(pLogicNode->pSubtable); - nodesDestroyList(pLogicNode->pAggFuncs); - break; - } - case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: { - SIndefRowsFuncLogicNode* pLogicNode = (SIndefRowsFuncLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pFuncs); - break; - } - case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: { - SInterpFuncLogicNode* pLogicNode = (SInterpFuncLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pFuncs); - nodesDestroyNode(pLogicNode->pFillValues); - nodesDestroyNode(pLogicNode->pTimeSeries); - break; - } - case QUERY_NODE_LOGIC_PLAN_GROUP_CACHE: { - SGroupCacheLogicNode* pLogicNode = (SGroupCacheLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - nodesDestroyList(pLogicNode->pGroupCols); - break; - } - case QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL: { - SDynQueryCtrlLogicNode* pLogicNode = (SDynQueryCtrlLogicNode*)pNode; - destroyLogicNode((SLogicNode*)pLogicNode); - break; - } - case QUERY_NODE_LOGIC_SUBPLAN: { - SLogicSubplan* pSubplan = (SLogicSubplan*)pNode; - nodesDestroyList(pSubplan->pChildren); - nodesDestroyNode((SNode*)pSubplan->pNode); - nodesClearList(pSubplan->pParents); - taosMemoryFreeClear(pSubplan->pVgroupList); - break; - } - case QUERY_NODE_LOGIC_PLAN: - nodesDestroyList(((SQueryLogicPlan*)pNode)->pTopSubplans); - break; - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - destroyScanPhysiNode((SScanPhysiNode*)pNode); - break; - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { - SLastRowScanPhysiNode* pPhyNode = (SLastRowScanPhysiNode*)pNode; - destroyScanPhysiNode((SScanPhysiNode*)pNode); - nodesDestroyList(pPhyNode->pGroupTags); - nodesDestroyList(pPhyNode->pTargets); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: { - STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; - destroyScanPhysiNode((SScanPhysiNode*)pNode); - nodesDestroyList(pPhyNode->pDynamicScanFuncs); - nodesDestroyList(pPhyNode->pGroupTags); - nodesDestroyList(pPhyNode->pTags); - nodesDestroyNode(pPhyNode->pSubtable); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { - SProjectPhysiNode* pPhyNode = (SProjectPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pProjections); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: { - SSortMergeJoinPhysiNode* pPhyNode = (SSortMergeJoinPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyNode(pPhyNode->pPrimKeyCond); - nodesDestroyNode(pPhyNode->pOtherOnCond); - nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pColEqCond); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: { - SHashJoinPhysiNode* pPhyNode = (SHashJoinPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pOnLeft); - nodesDestroyList(pPhyNode->pOnRight); - nodesDestroyNode(pPhyNode->pFilterConditions); - nodesDestroyList(pPhyNode->pTargets); - - nodesDestroyNode(pPhyNode->pPrimKeyCond); - nodesDestroyNode(pPhyNode->pColEqCond); - nodesDestroyNode(pPhyNode->pTagEqCond); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { - SAggPhysiNode* pPhyNode = (SAggPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pExprs); - nodesDestroyList(pPhyNode->pAggFuncs); - nodesDestroyList(pPhyNode->pGroupKeys); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { - SExchangePhysiNode* pPhyNode = (SExchangePhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pSrcEndPoints); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE: { - SMergePhysiNode* pPhyNode = (SMergePhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pMergeKeys); - nodesDestroyList(pPhyNode->pTargets); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_SORT: - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { - SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pExprs); - nodesDestroyList(pPhyNode->pSortKeys); - nodesDestroyList(pPhyNode->pTargets); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); - break; - case QUERY_NODE_PHYSICAL_PLAN_FILL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { - SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pFillExprs); - nodesDestroyList(pPhyNode->pNotFillExprs); - nodesDestroyNode(pPhyNode->pWStartTs); - nodesDestroyNode(pPhyNode->pValues); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); - break; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { - SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); - nodesDestroyNode(pPhyNode->pStateKey); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { - SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); - nodesDestroyNode(pPhyNode->pStartCond); - nodesDestroyNode(pPhyNode->pEndCond); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { - destroyPartitionPhysiNode((SPartitionPhysiNode*)pNode); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: { - SStreamPartitionPhysiNode* pPhyNode = (SStreamPartitionPhysiNode*)pNode; - destroyPartitionPhysiNode((SPartitionPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pTags); - nodesDestroyNode(pPhyNode->pSubtable); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: { - SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pExprs); - nodesDestroyList(pPhyNode->pFuncs); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: { - SInterpFuncPhysiNode* pPhyNode = (SInterpFuncPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pExprs); - nodesDestroyList(pPhyNode->pFuncs); - nodesDestroyNode(pPhyNode->pFillValues); - nodesDestroyNode(pPhyNode->pTimeSeries); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - destroyDataSinkNode((SDataSinkNode*)pNode); - break; - case QUERY_NODE_PHYSICAL_PLAN_INSERT: { - SDataInserterNode* pSink = (SDataInserterNode*)pNode; - destroyDataSinkNode((SDataSinkNode*)pSink); - taosMemoryFreeClear(pSink->pData); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { - SQueryInserterNode* pSink = (SQueryInserterNode*)pNode; - destroyDataSinkNode((SDataSinkNode*)pSink); - nodesDestroyList(pSink->pCols); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_DELETE: { - SDataDeleterNode* pSink = (SDataDeleterNode*)pNode; - destroyDataSinkNode((SDataSinkNode*)pSink); - nodesDestroyNode(pSink->pAffectedRows); - nodesDestroyNode(pSink->pStartTs); - nodesDestroyNode(pSink->pEndTs); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: { - SGroupCachePhysiNode* pPhyNode = (SGroupCachePhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pGroupCols); - break; - } - case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: { - SDynQueryCtrlPhysiNode* pPhyNode = (SDynQueryCtrlPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - break; - } - case QUERY_NODE_PHYSICAL_SUBPLAN: { - SSubplan* pSubplan = (SSubplan*)pNode; - nodesClearList(pSubplan->pChildren); - nodesDestroyNode((SNode*)pSubplan->pNode); - nodesDestroyNode((SNode*)pSubplan->pDataSink); - nodesDestroyNode((SNode*)pSubplan->pTagCond); - nodesDestroyNode((SNode*)pSubplan->pTagIndexCond); - nodesClearList(pSubplan->pParents); - break; - } - case QUERY_NODE_PHYSICAL_PLAN: - nodesDestroyList(((SQueryPlan*)pNode)->pSubplans); - break; - default: - break; - } - nodesFree(pNode); - return; -} SNodeList* nodesMakeList() { SNodeList* p = nodesCalloc(1, sizeof(SNodeList)); From 59a8fab0d76f93cc75169fd10da23ab474c66604 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 19 Nov 2023 17:11:26 +0800 Subject: [PATCH 03/43] adjust init funcNodes --- source/libs/nodes/src/nodesCodeFuncs.c | 404 ++++++++++++------------- 1 file changed, 202 insertions(+), 202 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 01d808c7d8..981e2bbaca 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -43,7 +43,7 @@ typedef struct SBuiltinNodeDefinition { FExecDestoryNode destoryFunc; } SBuiltinNodeDefinition; -SBuiltinNodeDefinition funcNodes[QUERY_NODE_END] = {}; +SBuiltinNodeDefinition funcNodes[QUERY_NODE_END] = {NULL}; static TdThreadOnce functionNodeInit = PTHREAD_ONCE_INIT; static int32_t initNodeCode = -1; @@ -7439,1407 +7439,1407 @@ void nodesDestroyNode(SNode* pNode) { // clang-format off static void doInitNodeFuncArray() { - setFunc("Column", + setFunc("Column", QUERY_NODE_COLUMN, sizeof(SColumnNode), columnNodeToJson, jsonToColumnNode, destroyColumnNode ); - setFunc("Value", + setFunc("Value", QUERY_NODE_VALUE, sizeof(SValueNode), valueNodeToJson, jsonToValueNode, destroyValueNode ); - setFunc("Operator", + setFunc("Operator", QUERY_NODE_OPERATOR, sizeof(SOperatorNode), operatorNodeToJson, jsonToOperatorNode, destroyOperatorNode ); - setFunc("LogicCondition", + setFunc("LogicCondition", QUERY_NODE_LOGIC_CONDITION, sizeof(SLogicConditionNode), logicConditionNodeToJson, jsonToLogicConditionNode, destoryLogicConditionNode ); - setFunc("Function", + setFunc("Function", QUERY_NODE_FUNCTION, sizeof(SFunctionNode), functionNodeToJson, jsonToFunctionNode, destoryFunctionNode ); - setFunc("RealTable", + setFunc("RealTable", QUERY_NODE_REAL_TABLE, sizeof(SRealTableNode), realTableNodeToJson, jsonToRealTableNode, destoryRealTableNode ); - setFunc("TempTable", + setFunc("TempTable", QUERY_NODE_TEMP_TABLE, sizeof(STempTableNode), tempTableNodeToJson, jsonToTempTableNode, destoryTempTableNode ); - setFunc("JoinTable", + setFunc("JoinTable", QUERY_NODE_JOIN_TABLE, sizeof(SJoinTableNode), joinTableNodeToJson, jsonToJoinTableNode, destoryJoinTableNode ); - setFunc("GroupingSet", + setFunc("GroupingSet", QUERY_NODE_GROUPING_SET, sizeof(SGroupingSetNode), groupingSetNodeToJson, jsonToGroupingSetNode, destoryGroupingSetNode ); - setFunc("OrderByExpr", + setFunc("OrderByExpr", QUERY_NODE_ORDER_BY_EXPR, sizeof(SOrderByExprNode), orderByExprNodeToJson, jsonToOrderByExprNode, destoryOrderByExprNode ); - setFunc("Limit", + setFunc("Limit", QUERY_NODE_LIMIT, sizeof(SLimitNode), limitNodeToJson, jsonToLimitNode, destoryXNode ); - setFunc("StateWindow", + setFunc("StateWindow", QUERY_NODE_STATE_WINDOW, sizeof(SStateWindowNode), stateWindowNodeToJson, jsonToStateWindowNode, destoryStateWindowNode ); - setFunc("SessionWinow", + setFunc("SessionWinow", QUERY_NODE_SESSION_WINDOW, sizeof(SSessionWindowNode), sessionWindowNodeToJson, jsonToSessionWindowNode, destorySessionWindowNode ); - setFunc("IntervalWindow", + setFunc("IntervalWindow", QUERY_NODE_INTERVAL_WINDOW, sizeof(SIntervalWindowNode), intervalWindowNodeToJson, jsonToIntervalWindowNode, destoryIntervalWindowNode ); - setFunc("NodeList", + setFunc("NodeList", QUERY_NODE_NODE_LIST, sizeof(SNodeListNode), nodeListNodeToJson, jsonToNodeListNode, destoryNodeListNode ); - setFunc("Fill", + setFunc("Fill", QUERY_NODE_FILL, sizeof(SFillNode), fillNodeToJson, jsonToFillNode, destoryFillNode ); - setFunc("RawExpr", + setFunc("RawExpr", QUERY_NODE_RAW_EXPR, sizeof(SRawExprNode), emptyNodeToJson, emptyJsonToNode, destoryRawExprNode ); - setFunc("Target", + setFunc("Target", QUERY_NODE_TARGET, sizeof(STargetNode), targetNodeToJson, jsonToTargetNode, destoryTargetNode ); - setFunc("DataBlockDesc", + setFunc("DataBlockDesc", QUERY_NODE_DATABLOCK_DESC, sizeof(SDataBlockDescNode), dataBlockDescNodeToJson, jsonToDataBlockDescNode, destoryDataBlockDescNode ); - setFunc("SlotDesc", + setFunc("SlotDesc", QUERY_NODE_SLOT_DESC, sizeof(SSlotDescNode), slotDescNodeToJson, jsonToSlotDescNode, destoryXNode ); - setFunc("ColumnDef", + setFunc("ColumnDef", QUERY_NODE_COLUMN_DEF, sizeof(SColumnDefNode), columnDefNodeToJson, jsonToColumnDefNode, destoryXNode ); - setFunc("DownstreamSource", + setFunc("DownstreamSource", QUERY_NODE_DOWNSTREAM_SOURCE, sizeof(SDownstreamSourceNode), downstreamSourceNodeToJson, jsonToDownstreamSourceNode, destoryXNode ); - setFunc("DatabaseOptions", + setFunc("DatabaseOptions", QUERY_NODE_DATABASE_OPTIONS, sizeof(SDatabaseOptions), databaseOptionsToJson, jsonToDatabaseOptions, destoryDatabaseOptions ); - setFunc("TableOptions", + setFunc("TableOptions", QUERY_NODE_TABLE_OPTIONS, sizeof(STableOptions), tableOptionsToJson, jsonToTableOptions, destoryTableOptions ); - setFunc("IndexOptions", + setFunc("IndexOptions", QUERY_NODE_INDEX_OPTIONS, sizeof(SIndexOptions), indexOptionsToJson, jsonToIndexOptions, destoryIndexOptions ); - setFunc("ExplainOptions", + setFunc("ExplainOptions", QUERY_NODE_EXPLAIN_OPTIONS, sizeof(SExplainOptions), explainOptionsToJson, jsonToExplainOptions, destoryXNode ); - setFunc("StreamOptions", + setFunc("StreamOptions", QUERY_NODE_STREAM_OPTIONS, sizeof(SStreamOptions), streamOptionsToJson, jsonToStreamOptions, destoryStreamOptions ); - setFunc("LeftValue", + setFunc("LeftValue", QUERY_NODE_LEFT_VALUE, sizeof(SLeftValueNode), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("ColumnRef", + setFunc("ColumnRef", QUERY_NODE_COLUMN_REF, sizeof(SColumnDefNode), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("WhenThen", + setFunc("WhenThen", QUERY_NODE_WHEN_THEN, sizeof(SWhenThenNode), whenThenNodeToJson, jsonToWhenThenNode, destoryWhenThenNode ); - setFunc("CaseWhen", + setFunc("CaseWhen", QUERY_NODE_CASE_WHEN, sizeof(SCaseWhenNode), caseWhenNodeToJson, jsonToCaseWhenNode, destoryCaseWhenNode ); - setFunc("EventWindow", + setFunc("EventWindow", QUERY_NODE_EVENT_WINDOW, sizeof(SEventWindowNode), eventWindowNodeToJson, jsonToEventWindowNode, destoryEventWindowNode ); - setFunc("HintNode", + setFunc("HintNode", QUERY_NODE_HINT, sizeof(SHintNode), emptyNodeToJson, emptyJsonToNode, destoryHintNode ); - setFunc("ViewNode", + setFunc("ViewNode", QUERY_NODE_VIEW, sizeof(SViewNode), emptyNodeToJson, emptyJsonToNode, destoryViewNode ); - setFunc("SetOperator", + setFunc("SetOperator", QUERY_NODE_SET_OPERATOR, sizeof(SSetOperator), setOperatorToJson, jsonToSetOperator, destorySetOperator ); - setFunc("SelectStmt", + setFunc("SelectStmt", QUERY_NODE_SELECT_STMT, sizeof(SSelectStmt), selectStmtToJson, jsonToSelectStmt, destorySelectStmt ); - setFunc("VnodeModifyStmt", + setFunc("VnodeModifyStmt", QUERY_NODE_VNODE_MODIFY_STMT, sizeof(SVnodeModifyOpStmt), vnodeModifyStmtToJson, jsonToVnodeModifyStmt, destoryVnodeModifyOpStmt ); - setFunc("CreateDatabaseStmt", + setFunc("CreateDatabaseStmt", QUERY_NODE_CREATE_DATABASE_STMT, sizeof(SCreateDatabaseStmt), createDatabaseStmtToJson, jsonToCreateDatabaseStmt, destoryCreateDatabaseStmt ); - setFunc("DropDatabaseStmt", + setFunc("DropDatabaseStmt", QUERY_NODE_DROP_DATABASE_STMT, sizeof(SDropDatabaseStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("AlterDatabaseStmt", + setFunc("AlterDatabaseStmt", QUERY_NODE_ALTER_DATABASE_STMT, sizeof(SAlterDatabaseStmt), alterDatabaseStmtToJson, jsonToAlterDatabaseStmt, destoryAlterDatabaseStmt ); - setFunc("FlushDatabaseStmt", + setFunc("FlushDatabaseStmt", QUERY_NODE_FLUSH_DATABASE_STMT, sizeof(SFlushDatabaseStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("TrimDatabaseStmt", + setFunc("TrimDatabaseStmt", QUERY_NODE_TRIM_DATABASE_STMT, sizeof(STrimDatabaseStmt), trimDatabaseStmtToJson, jsonToTrimDatabaseStmt, destoryXNode ); - setFunc("CreateTableStmt", + setFunc("CreateTableStmt", QUERY_NODE_CREATE_TABLE_STMT, sizeof(SCreateTableStmt), createTableStmtToJson, jsonToCreateTableStmt, destoryCreateTableStmt ); - setFunc("CreateSubtableClause", + setFunc("CreateSubtableClause", QUERY_NODE_CREATE_SUBTABLE_CLAUSE, sizeof(SCreateSubTableClause), createSubTableClauseToJson, jsonToCreateSubTableClause, destoryCreateSubTableClause ); - setFunc("CreateMultiTableStmt", + setFunc("CreateMultiTableStmt", QUERY_NODE_CREATE_MULTI_TABLES_STMT, sizeof(SCreateMultiTablesStmt), createMultiTablesStmtToJson, jsonToCreateMultiTablesStmt, destoryCreateMultiTablesStmt ); - setFunc("DropTableClause", + setFunc("DropTableClause", QUERY_NODE_DROP_TABLE_CLAUSE, sizeof(SDropTableClause), dropTableClauseToJson, jsonToDropTableClause, destoryXNode ); - setFunc("DropTableStmt", + setFunc("DropTableStmt", QUERY_NODE_DROP_TABLE_STMT, sizeof(SDropTableStmt), dropTableStmtToJson, jsonToDropTableStmt, destoryDropTableStmt ); - setFunc("DropSuperTableStmt", + setFunc("DropSuperTableStmt", QUERY_NODE_DROP_SUPER_TABLE_STMT, sizeof(SDropSuperTableStmt), dropStableStmtToJson, jsonToDropStableStmt, destoryXNode ); - setFunc("AlterTableStmt", + setFunc("AlterTableStmt", QUERY_NODE_ALTER_TABLE_STMT, sizeof(SAlterTableStmt), alterTableStmtToJson, jsonToAlterTableStmt, destoryAlterTableStmt ); - setFunc("AlterSuperTableStmt", + setFunc("AlterSuperTableStmt", QUERY_NODE_ALTER_SUPER_TABLE_STMT, sizeof(SAlterTableStmt), alterStableStmtToJson, jsonToAlterStableStmt, destoryAlterTableStmt ); - setFunc("CreateUserStmt", + setFunc("CreateUserStmt", QUERY_NODE_CREATE_USER_STMT, sizeof(SCreateUserStmt), createUserStmtToJson, jsonToCreateUserStmt, destoryCreateUserStmt ); - setFunc("AlterUserStmt", + setFunc("AlterUserStmt", QUERY_NODE_ALTER_USER_STMT, sizeof(SAlterUserStmt), alterUserStmtToJson, jsonToAlterUserStmt, destoryAlterUserStmt ); - setFunc("DropUserStmt", + setFunc("DropUserStmt", QUERY_NODE_DROP_USER_STMT, sizeof(SDropUserStmt), dropUserStmtToJson, jsonToDropUserStmt, destoryXNode ); - setFunc("UseDatabaseStmt", + setFunc("UseDatabaseStmt", QUERY_NODE_USE_DATABASE_STMT, sizeof(SUseDatabaseStmt), useDatabaseStmtToJson, jsonToUseDatabaseStmt, destoryXNode ); - setFunc("CreateDnodeStmt", + setFunc("CreateDnodeStmt", QUERY_NODE_CREATE_DNODE_STMT, sizeof(SCreateDnodeStmt), createDnodeStmtToJson, jsonToCreateDnodeStmt, destoryXNode ); - setFunc("DropDnodeStmt", + setFunc("DropDnodeStmt", QUERY_NODE_DROP_DNODE_STMT, sizeof(SDropDnodeStmt), dropDnodeStmtToJson, jsonToDropDnodeStmt, destoryXNode ); - setFunc("AlterDnodeStmt", + setFunc("AlterDnodeStmt", QUERY_NODE_ALTER_DNODE_STMT, sizeof(SAlterDnodeStmt), alterDnodeStmtToJson, jsonToAlterDnodeStmt, destoryXNode ); - setFunc("CreateIndexStmt", + setFunc("CreateIndexStmt", QUERY_NODE_CREATE_INDEX_STMT, sizeof(SCreateIndexStmt), createIndexStmtToJson, jsonToCreateIndexStmt, destoryCreateIndexStmt ); - setFunc("DropIndexStmt", + setFunc("DropIndexStmt", QUERY_NODE_DROP_INDEX_STMT, sizeof(SDropIndexStmt), dropIndexStmtToJson, jsonToDropIndexStmt, destoryXNode ); - setFunc("CreateQnodeStmt", + setFunc("CreateQnodeStmt", QUERY_NODE_CREATE_QNODE_STMT, sizeof(SCreateComponentNodeStmt), createQnodeStmtToJson, jsonToCreateQnodeStmt, destoryXNode ); - setFunc("DropQnodeStmt", + setFunc("DropQnodeStmt", QUERY_NODE_DROP_QNODE_STMT, sizeof(SDropComponentNodeStmt), dropQnodeStmtToJson, jsonToDropQnodeStmt, destoryXNode ); - setFunc("CreateBnodeStmt", + setFunc("CreateBnodeStmt", QUERY_NODE_CREATE_BNODE_STMT, sizeof(SCreateComponentNodeStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("DropBnodeStmt", + setFunc("DropBnodeStmt", QUERY_NODE_DROP_BNODE_STMT, sizeof(SDropComponentNodeStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("CreateSnodeStmt", + setFunc("CreateSnodeStmt", QUERY_NODE_CREATE_SNODE_STMT, sizeof(SCreateComponentNodeStmt), createSnodeStmtToJson, jsonToCreateSnodeStmt, destoryXNode ); - setFunc("DropSnodeStmt", + setFunc("DropSnodeStmt", QUERY_NODE_DROP_SNODE_STMT, sizeof(SDropComponentNodeStmt), dropSnodeStmtToJson, jsonToDropSnodeStmt, destoryXNode ); - setFunc("CreateMnodeStmt", + setFunc("CreateMnodeStmt", QUERY_NODE_CREATE_MNODE_STMT, sizeof(SCreateComponentNodeStmt), createMnodeStmtToJson, jsonToCreateMnodeStmt, destoryXNode ); - setFunc("DropMnodeStmt", + setFunc("DropMnodeStmt", QUERY_NODE_DROP_MNODE_STMT, sizeof(SDropComponentNodeStmt), dropMnodeStmtToJson, jsonToDropMnodeStmt, destoryXNode ); - setFunc("CreateTopicStmt", + setFunc("CreateTopicStmt", QUERY_NODE_CREATE_TOPIC_STMT, sizeof(SCreateTopicStmt), createTopicStmtToJson, jsonToCreateTopicStmt, destoryCreateTopicStmt ); - setFunc("DropTopicStmt", + setFunc("DropTopicStmt", QUERY_NODE_DROP_TOPIC_STMT, sizeof(SDropTopicStmt), dropTopicStmtToJson, jsonToDropTopicStmt, destoryXNode ); - setFunc("DropConsumerGroupStmt", + setFunc("DropConsumerGroupStmt", QUERY_NODE_DROP_CGROUP_STMT, sizeof(SDropCGroupStmt), dropConsumerGroupStmtToJson, jsonToDropConsumerGroupStmt, destoryXNode ); - setFunc("AlterLocalStmt", + setFunc("AlterLocalStmt", QUERY_NODE_ALTER_LOCAL_STMT, sizeof(SAlterLocalStmt), alterLocalStmtToJson, jsonToAlterLocalStmt, destoryXNode ); - setFunc("ExplainStmt", + setFunc("ExplainStmt", QUERY_NODE_EXPLAIN_STMT, sizeof(SExplainStmt), explainStmtToJson, jsonToExplainStmt, destoryExplainStmt ); - setFunc("DescribeStmt", + setFunc("DescribeStmt", QUERY_NODE_DESCRIBE_STMT, sizeof(SDescribeStmt), describeStmtToJson, jsonToDescribeStmt, destoryDescribeStmt ); - setFunc("QueryCacheStmt", + setFunc("QueryCacheStmt", QUERY_NODE_RESET_QUERY_CACHE_STMT, sizeof(SNode), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("CompactDatabaseStmt", + setFunc("CompactDatabaseStmt", QUERY_NODE_COMPACT_DATABASE_STMT, sizeof(SCompactDatabaseStmt), compactDatabaseStmtToJson, jsonToCompactDatabaseStmt, destoryCompactDatabaseStmt ); - setFunc("CreateFunctionStmt", + setFunc("CreateFunctionStmt", QUERY_NODE_CREATE_FUNCTION_STMT, sizeof(SCreateFunctionStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("DropFunctionStmt", + setFunc("DropFunctionStmt", QUERY_NODE_DROP_FUNCTION_STMT, sizeof(SDropFunctionStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("CreateStreamStmt", + setFunc("CreateStreamStmt", QUERY_NODE_CREATE_STREAM_STMT, sizeof(SCreateStreamStmt), createStreamStmtToJson, jsonToCreateStreamStmt, destoryCreateStreamStmt ); - setFunc("DropStreamStmt", + setFunc("DropStreamStmt", QUERY_NODE_DROP_STREAM_STMT, sizeof(SDropStreamStmt), dropStreamStmtToJson, jsonToDropStreamStmt, destoryXNode ); - setFunc("PauseStreamStmt", + setFunc("PauseStreamStmt", QUERY_NODE_PAUSE_STREAM_STMT, sizeof(SPauseStreamStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("ResumeStreamStmt", + setFunc("ResumeStreamStmt", QUERY_NODE_RESUME_STREAM_STMT, sizeof(SResumeStreamStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("BalanceVgroupStmt", + setFunc("BalanceVgroupStmt", QUERY_NODE_BALANCE_VGROUP_STMT, sizeof(SBalanceVgroupStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("BalanceVgroupLeaderStmt", + setFunc("BalanceVgroupLeaderStmt", QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, sizeof(SBalanceVgroupLeaderStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("MergeVgroupStmt", + setFunc("MergeVgroupStmt", QUERY_NODE_MERGE_VGROUP_STMT, sizeof(SMergeVgroupStmt), mergeVgroupStmtToJson, jsonToMergeVgroupStmt, destoryXNode ); - setFunc("RedistributeVgroupStmt", + setFunc("RedistributeVgroupStmt", QUERY_NODE_REDISTRIBUTE_VGROUP_STMT, sizeof(SRedistributeVgroupStmt), redistributeVgroupStmtToJson, jsonToRedistributeVgroupStmt, destoryRedistributeVgroupStmt ); - setFunc("SplitVgroupStmt", + setFunc("SplitVgroupStmt", QUERY_NODE_SPLIT_VGROUP_STMT, sizeof(SSplitVgroupStmt), splitVgroupStmtToJson, jsonToSplitVgroupStmt, destoryXNode ); - setFunc("SyncDBStmt", + setFunc("SyncDBStmt", QUERY_NODE_SYNCDB_STMT, 0, emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("GrantStmt", + setFunc("GrantStmt", QUERY_NODE_GRANT_STMT, sizeof(SGrantStmt), grantStmtToJson, jsonToGrantStmt, destoryGrantStmt ); - setFunc("RevokeStmt", + setFunc("RevokeStmt", QUERY_NODE_REVOKE_STMT, sizeof(SRevokeStmt), revokeStmtToJson, jsonToRevokeStmt, destoryRevokeStmt ); - setFunc("ShowDnodesStmt", + setFunc("ShowDnodesStmt", QUERY_NODE_SHOW_DNODES_STMT, sizeof(SShowStmt), showDnodesStmtToJson, jsonToShowDnodesStmt, destoryShowStmt ); - setFunc("ShowMnodesStmt", + setFunc("ShowMnodesStmt", QUERY_NODE_SHOW_MNODES_STMT, sizeof(SShowStmt), showMnodesStmtToJson, jsonToShowMnodesStmt, destoryShowStmt ); - setFunc("ShowModulesStmt", + setFunc("ShowModulesStmt", QUERY_NODE_SHOW_MODULES_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowQnodesStmt", + setFunc("ShowQnodesStmt", QUERY_NODE_SHOW_QNODES_STMT, sizeof(SShowStmt), showQnodesStmtToJson, jsonToShowQnodesStmt, destoryShowStmt ); - setFunc("ShowSnodesStmt", + setFunc("ShowSnodesStmt", QUERY_NODE_SHOW_SNODES_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowBnodesStmt", + setFunc("ShowBnodesStmt", QUERY_NODE_SHOW_BNODES_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowClusterStmt", + setFunc("ShowClusterStmt", QUERY_NODE_SHOW_CLUSTER_STMT, sizeof(SShowStmt), showClusterStmtToJson, jsonToShowClusterStmt, destoryShowStmt ); - setFunc("ShowDatabaseStmt", + setFunc("ShowDatabaseStmt", QUERY_NODE_SHOW_DATABASES_STMT, sizeof(SShowStmt), showDatabasesStmtToJson, jsonToShowDatabasesStmt, destoryShowStmt ); - setFunc("ShowFunctionsStmt", + setFunc("ShowFunctionsStmt", QUERY_NODE_SHOW_FUNCTIONS_STMT, sizeof(SShowStmt), showFunctionsStmtToJson, jsonToShowFunctionsStmt, destoryShowStmt ); - setFunc("ShowIndexesStmt", + setFunc("ShowIndexesStmt", QUERY_NODE_SHOW_INDEXES_STMT, sizeof(SShowStmt), showIndexesStmtToJson, jsonToShowIndexesStmt, destoryShowStmt ); - setFunc("ShowStablesStmt", + setFunc("ShowStablesStmt", QUERY_NODE_SHOW_STABLES_STMT, sizeof(SShowStmt), showStablesStmtToJson, jsonToShowStablesStmt, destoryShowStmt ); - setFunc("ShowStreamsStmt", + setFunc("ShowStreamsStmt", QUERY_NODE_SHOW_STREAMS_STMT, sizeof(SShowStmt), showStreamsStmtToJson, jsonToShowStreamsStmt, destoryShowStmt ); - setFunc("ShowTablesStmt", + setFunc("ShowTablesStmt", QUERY_NODE_SHOW_TABLES_STMT, sizeof(SShowStmt), showTablesStmtToJson, jsonToShowTablesStmt, destoryShowStmt ); - setFunc("ShowTagsStmt", + setFunc("ShowTagsStmt", QUERY_NODE_SHOW_TAGS_STMT, sizeof(SShowStmt), showTagsStmtToJson, jsonToShowTagsStmt, destoryShowStmt ); - setFunc("ShowUsersStmt", + setFunc("ShowUsersStmt", QUERY_NODE_SHOW_USERS_STMT, sizeof(SShowStmt), showUsersStmtToJson, jsonToShowUsersStmt, destoryShowStmt ); - setFunc("ShowLicencesStmt", + setFunc("ShowLicencesStmt", QUERY_NODE_SHOW_LICENCES_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowVgroupsStmt", + setFunc("ShowVgroupsStmt", QUERY_NODE_SHOW_VGROUPS_STMT, sizeof(SShowStmt), showVgroupsStmtToJson, jsonToShowVgroupsStmt, destoryShowStmt ); - setFunc("ShowTopicsStmt", + setFunc("ShowTopicsStmt", QUERY_NODE_SHOW_TOPICS_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowConsumersStmt", + setFunc("ShowConsumersStmt", QUERY_NODE_SHOW_CONSUMERS_STMT, sizeof(SShowStmt), showConsumersStmtToJson, jsonToShowConsumersStmt, destoryShowStmt ); - setFunc("ShowQueriesStmt", + setFunc("ShowQueriesStmt", QUERY_NODE_SHOW_QUERIES_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowConnectionsStmt", + setFunc("ShowConnectionsStmt", QUERY_NODE_SHOW_CONNECTIONS_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowAppsStmt", + setFunc("ShowAppsStmt", QUERY_NODE_SHOW_APPS_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowVariablesStmt", + setFunc("ShowVariablesStmt", QUERY_NODE_SHOW_VARIABLES_STMT, sizeof(SShowStmt), showVariablesStmtToJson, jsonToShowVariablesStmt, destoryShowStmt ); - setFunc("ShowDnodeVariablesStmt", + setFunc("ShowDnodeVariablesStmt", QUERY_NODE_SHOW_DNODE_VARIABLES_STMT, sizeof(SShowDnodeVariablesStmt), showDnodeVariablesStmtToJson, jsonToShowDnodeVariablesStmt, destoryShowDnodeVariablesStmt ); - setFunc("ShowTransactionsStmt", + setFunc("ShowTransactionsStmt", QUERY_NODE_SHOW_TRANSACTIONS_STMT, sizeof(SShowStmt), showTransactionsStmtToJson, jsonToShowTransactionsStmt, destoryShowStmt ); - setFunc("ShowSubscriptionsStmt", + setFunc("ShowSubscriptionsStmt", QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, sizeof(SShowStmt), showSubscriptionsStmtToJson, jsonToShowSubscriptionsStmt, destoryShowStmt ); - setFunc("ShowVnodeStmt", + setFunc("ShowVnodeStmt", QUERY_NODE_SHOW_VNODES_STMT, sizeof(SShowStmt), showVnodesStmtToJson, jsonToShowVnodesStmt, destoryShowStmt ); - setFunc("ShowUserPrivilegesStmt", + setFunc("ShowUserPrivilegesStmt", QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, sizeof(SShowStmt), showUserPrivilegesStmtToJson, jsonToShowUserPrivilegesStmt, destoryShowStmt ); - setFunc("ShowViewsStmt", + setFunc("ShowViewsStmt", QUERY_NODE_SHOW_VIEWS_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowCreateViewStmt", + setFunc("ShowCreateViewStmt", QUERY_NODE_SHOW_CREATE_VIEW_STMT, sizeof(SShowCreateViewStmt), showCreateViewStmtToJson, jsonToShowCreateViewStmt, destoryXNode ); - setFunc("ShowCreateDatabasesStmt", + setFunc("ShowCreateDatabasesStmt", QUERY_NODE_SHOW_CREATE_DATABASE_STMT, sizeof(SShowCreateDatabaseStmt), showCreateDatabaseStmtToJson, jsonToShowCreateDatabaseStmt, destoryShowCreateDatabaseStmt ); - setFunc("ShowCreateTablesStmt", + setFunc("ShowCreateTablesStmt", QUERY_NODE_SHOW_CREATE_TABLE_STMT, sizeof(SShowCreateTableStmt), showCreateTableStmtToJson, jsonToShowCreateTableStmt, destoryShowCreateTableStmt ); - setFunc("ShowCreateStablesStmt", + setFunc("ShowCreateStablesStmt", QUERY_NODE_SHOW_CREATE_STABLE_STMT, sizeof(SShowCreateTableStmt), showCreateStableStmtToJson, jsonToShowCreateStableStmt, destoryShowCreateTableStmt ); - setFunc("ShowTableDistributedStmt", + setFunc("ShowTableDistributedStmt", QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, sizeof(SShowTableDistributedStmt), showTableDistributedStmtToJson, jsonToShowTableDistributedStmt, destoryXNode ); - setFunc("ShowLocalVariablesStmt", + setFunc("ShowLocalVariablesStmt", QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, sizeof(SShowStmt), showLocalVariablesStmtToJson, jsonToShowLocalVariablesStmt, destoryShowStmt ); - setFunc("ShowScoresStmt", + setFunc("ShowScoresStmt", QUERY_NODE_SHOW_SCORES_STMT, sizeof(SShowStmt), emptyNodeToJson, emptyJsonToNode, destoryShowStmt ); - setFunc("ShowTableTagsStmt", + setFunc("ShowTableTagsStmt", QUERY_NODE_SHOW_TABLE_TAGS_STMT, sizeof(SShowTableTagsStmt), showTableTagsStmtToJson, jsonToShowTableTagsStmt, destoryShowTableTagsStmt ); - setFunc("KillConnectionStmt", + setFunc("KillConnectionStmt", QUERY_NODE_KILL_CONNECTION_STMT, sizeof(SKillStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("KillQueryStmt", + setFunc("KillQueryStmt", QUERY_NODE_KILL_QUERY_STMT, sizeof(SKillQueryStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("KillTransactionStmt", + setFunc("KillTransactionStmt", QUERY_NODE_KILL_TRANSACTION_STMT, sizeof(SKillStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("DeleteStmt", + setFunc("DeleteStmt", QUERY_NODE_DELETE_STMT, sizeof(SDeleteStmt), deleteStmtToJson, jsonToDeleteStmt, destoryDeleteStmt ); - setFunc("InsertStmt", + setFunc("InsertStmt", QUERY_NODE_INSERT_STMT, sizeof(SInsertStmt), insertStmtToJson, jsonToInsertStmt, destoryInsertStmt ); - setFunc("QueryNode", + setFunc("QueryNode", QUERY_NODE_QUERY, sizeof(SQuery), emptyNodeToJson, emptyJsonToNode, destoryQueryNode ); - setFunc("ShowDbAliveStmt", + setFunc("ShowDbAliveStmt", QUERY_NODE_SHOW_DB_ALIVE_STMT, sizeof(SShowAliveStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("ShowClusterAliveStmt", + setFunc("ShowClusterAliveStmt", QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, sizeof(SShowAliveStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("RestoreDnodeStmt", + setFunc("RestoreDnodeStmt", QUERY_NODE_RESTORE_DNODE_STMT, sizeof(SRestoreComponentNodeStmt), emptyNodeToJson, jsonToRestoreDnodeStmt, destoryXNode ); - setFunc("RestoreQnodeStmt", + setFunc("RestoreQnodeStmt", QUERY_NODE_RESTORE_QNODE_STMT, sizeof(SRestoreComponentNodeStmt), emptyNodeToJson, jsonToRestoreQnodeStmt, destoryXNode ); - setFunc("RestoreMnodeStmt", + setFunc("RestoreMnodeStmt", QUERY_NODE_RESTORE_MNODE_STMT, sizeof(SRestoreComponentNodeStmt), emptyNodeToJson, jsonToRestoreMnodeStmt, destoryXNode ); - setFunc("RestoreVnodeStmt", + setFunc("RestoreVnodeStmt", QUERY_NODE_RESTORE_VNODE_STMT, sizeof(SRestoreComponentNodeStmt), emptyNodeToJson, jsonToRestoreVnodeStmt, destoryXNode ); - setFunc("CreateViewStmt", + setFunc("CreateViewStmt", QUERY_NODE_CREATE_VIEW_STMT, sizeof(SCreateViewStmt), emptyNodeToJson, emptyJsonToNode, destoryCreateViewStmt ); - setFunc("DropViewStmt", + setFunc("DropViewStmt", QUERY_NODE_DROP_VIEW_STMT, sizeof(SDropViewStmt), emptyNodeToJson, emptyJsonToNode, destoryXNode ); - setFunc("LogicScan", + setFunc("LogicScan", QUERY_NODE_LOGIC_PLAN_SCAN, sizeof(SScanLogicNode), logicScanNodeToJson, jsonToLogicScanNode, destoryScanLogicNode ); - setFunc("LogicJoin", + setFunc("LogicJoin", QUERY_NODE_LOGIC_PLAN_JOIN, sizeof(SJoinLogicNode), logicJoinNodeToJson, jsonToLogicJoinNode, destoryJoinLogicNode ); - setFunc("LogicAgg", + setFunc("LogicAgg", QUERY_NODE_LOGIC_PLAN_AGG, sizeof(SAggLogicNode), logicAggNodeToJson, jsonToLogicAggNode, destoryAggLogicNode ); - setFunc("LogicProject", + setFunc("LogicProject", QUERY_NODE_LOGIC_PLAN_PROJECT, sizeof(SProjectLogicNode), logicProjectNodeToJson, jsonToLogicProjectNode, destoryProjectLogicNode ); - setFunc("LogicVnodeModify", + setFunc("LogicVnodeModify", QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY, sizeof(SVnodeModifyLogicNode), logicVnodeModifyNodeToJson, jsonToLogicVnodeModifyNode, destoryVnodeModifyLogicNode ); - setFunc("LogicExchange", + setFunc("LogicExchange", QUERY_NODE_LOGIC_PLAN_EXCHANGE, sizeof(SExchangeLogicNode), logicExchangeNodeToJson, jsonToLogicExchangeNode, destoryExchangeLogicNode ); - setFunc("LogicMerge", + setFunc("LogicMerge", QUERY_NODE_LOGIC_PLAN_MERGE, sizeof(SMergeLogicNode), logicMergeNodeToJson, jsonToLogicMergeNode, destoryMergeLogicNode ); - setFunc("LogicWindow", + setFunc("LogicWindow", QUERY_NODE_LOGIC_PLAN_WINDOW, sizeof(SWindowLogicNode), logicWindowNodeToJson, jsonToLogicWindowNode, destoryWindowLogicNode ); - setFunc("LogicFill", + setFunc("LogicFill", QUERY_NODE_LOGIC_PLAN_FILL, sizeof(SFillLogicNode), logicFillNodeToJson, jsonToLogicFillNode, destoryFillLogicNode ); - setFunc("LogicSort", + setFunc("LogicSort", QUERY_NODE_LOGIC_PLAN_SORT, sizeof(SSortLogicNode), logicSortNodeToJson, jsonToLogicSortNode, destorySortLogicNode ); - setFunc("LogicPartition", + setFunc("LogicPartition", QUERY_NODE_LOGIC_PLAN_PARTITION, sizeof(SPartitionLogicNode), logicPartitionNodeToJson, jsonToLogicPartitionNode, destoryPartitionLogicNode ); - setFunc("LogicIndefRowsFunc", + setFunc("LogicIndefRowsFunc", QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC, sizeof(SIndefRowsFuncLogicNode), logicIndefRowsFuncNodeToJson, jsonToLogicIndefRowsFuncNode, destoryIndefRowsFuncLogicNode ); - setFunc("LogicInterpFunc", + setFunc("LogicInterpFunc", QUERY_NODE_LOGIC_PLAN_INTERP_FUNC, sizeof(SInterpFuncLogicNode), logicInterpFuncNodeToJson, jsonToLogicInterpFuncNode, destoryInterpFuncLogicNode ); - setFunc("LogicGroupCache", + setFunc("LogicGroupCache", QUERY_NODE_LOGIC_PLAN_GROUP_CACHE, sizeof(SGroupCacheLogicNode), logicGroupCacheNodeToJson, jsonToLogicGroupCacheNode, destoryGroupCacheLogicNode ); - setFunc("LogicDynamicQueryCtrl", + setFunc("LogicDynamicQueryCtrl", QUERY_NODE_LOGIC_PLAN_DYN_QUERY_CTRL, sizeof(SDynQueryCtrlLogicNode), logicDynQueryCtrlNodeToJson, jsonToLogicDynQueryCtrlNode, destoryDynQueryCtrlLogicNode ); - setFunc("LogicSubplan", + setFunc("LogicSubplan", QUERY_NODE_LOGIC_SUBPLAN, sizeof(SLogicSubplan), logicSubplanToJson, jsonToLogicSubplan, destoryLogicSubplan ); - setFunc("LogicPlan", + setFunc("LogicPlan", QUERY_NODE_LOGIC_PLAN, sizeof(SQueryLogicPlan), logicPlanToJson, jsonToLogicPlan, destoryQueryLogicPlan ); - setFunc("PhysiTagScan", + setFunc("PhysiTagScan", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, sizeof(STagScanPhysiNode), physiTagScanNodeToJson, jsonToPhysiTagScanNode, destroyScanPhysiNode ); - setFunc("PhysiTableScan", + setFunc("PhysiTableScan", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, sizeof(STableScanPhysiNode), physiTableScanNodeToJson, jsonToPhysiTableScanNode, destoryTableScanPhysiNode ); - setFunc("PhysiTableSeqScan", + setFunc("PhysiTableSeqScan", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, sizeof(STableSeqScanPhysiNode), emptyNodeToJson, emptyJsonToNode, destoryTableScanPhysiNode ); - setFunc("PhysiTableMergeScan", + setFunc("PhysiTableMergeScan", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, sizeof(STableMergeScanPhysiNode), physiTableScanNodeToJson, jsonToPhysiTableScanNode, destoryTableScanPhysiNode ); - setFunc("PhysiSreamScan", + setFunc("PhysiSreamScan", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, sizeof(SStreamScanPhysiNode), physiTableScanNodeToJson, jsonToPhysiTableScanNode, destoryTableScanPhysiNode ); - setFunc("PhysiSystemTableScan", + setFunc("PhysiSystemTableScan", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, sizeof(SSystemTableScanPhysiNode), physiSysTableScanNodeToJson, jsonToPhysiSysTableScanNode, destroyScanPhysiNode ); - setFunc("PhysiBlockDistScan", + setFunc("PhysiBlockDistScan", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, sizeof(SBlockDistScanPhysiNode), physiScanNodeToJson, jsonToPhysiScanNode, destroyScanPhysiNode ); - setFunc("PhysiLastRowScan", + setFunc("PhysiLastRowScan", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, sizeof(SLastRowScanPhysiNode), physiLastRowScanNodeToJson, jsonToPhysiLastRowScanNode, destoryLastRowScanPhysiNode ); - setFunc("PhysiProject", + setFunc("PhysiProject", QUERY_NODE_PHYSICAL_PLAN_PROJECT, sizeof(SProjectPhysiNode), physiProjectNodeToJson, jsonToPhysiProjectNode, destoryProjectPhysiNode ); - setFunc("PhysiMergeJoin", + setFunc("PhysiMergeJoin", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, sizeof(SSortMergeJoinPhysiNode), physiMergeJoinNodeToJson, jsonToPhysiMergeJoinNode, destorySortMergeJoinPhysiNode ); - setFunc("PhysiAgg", + setFunc("PhysiAgg", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, sizeof(SAggPhysiNode), physiAggNodeToJson, jsonToPhysiAggNode, destoryAggPhysiNode ); - setFunc("PhysiExchange", + setFunc("PhysiExchange", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, sizeof(SExchangePhysiNode), physiExchangeNodeToJson, jsonToPhysiExchangeNode, destoryExchangePhysiNode ); - setFunc("PhysiMerge", + setFunc("PhysiMerge", QUERY_NODE_PHYSICAL_PLAN_MERGE, sizeof(SMergePhysiNode), physiMergeNodeToJson, jsonToPhysiMergeNode, destoryMergePhysiNode ); - setFunc("PhysiSort", + setFunc("PhysiSort", QUERY_NODE_PHYSICAL_PLAN_SORT, sizeof(SSortPhysiNode), physiSortNodeToJson, jsonToPhysiSortNode, destorySortPhysiNode ); - setFunc("PhysiGroupSort", + setFunc("PhysiGroupSort", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, sizeof(SGroupSortPhysiNode), physiSortNodeToJson, jsonToPhysiSortNode, destorySortPhysiNode ); - setFunc("PhysiHashInterval", + setFunc("PhysiHashInterval", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, sizeof(SIntervalPhysiNode), physiIntervalNodeToJson, jsonToPhysiIntervalNode, destroyWindowPhysiNode ); - setFunc("PhysiMergeAlignedInterval", + setFunc("PhysiMergeAlignedInterval", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, sizeof(SMergeAlignedIntervalPhysiNode), physiIntervalNodeToJson, jsonToPhysiIntervalNode, destroyWindowPhysiNode ); - setFunc("PhysiStreamInterval", + setFunc("PhysiStreamInterval", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, sizeof(SStreamIntervalPhysiNode), physiIntervalNodeToJson, jsonToPhysiIntervalNode, destroyWindowPhysiNode ); - setFunc("PhysiStreamFinalInterval", + setFunc("PhysiStreamFinalInterval", QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, sizeof(SStreamFinalIntervalPhysiNode), physiIntervalNodeToJson, jsonToPhysiIntervalNode, destroyWindowPhysiNode ); - setFunc("PhysiStreamSemiInterval", + setFunc("PhysiStreamSemiInterval", QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, sizeof(SStreamSemiIntervalPhysiNode), physiIntervalNodeToJson, jsonToPhysiIntervalNode, destroyWindowPhysiNode ); - setFunc("PhysiFill", + setFunc("PhysiFill", QUERY_NODE_PHYSICAL_PLAN_FILL, sizeof(SFillPhysiNode), physiFillNodeToJson, jsonToPhysiFillNode, destoryFillPhysiNode ); - setFunc("PhysiStreamFill", + setFunc("PhysiStreamFill", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, sizeof(SFillPhysiNode), physiFillNodeToJson, jsonToPhysiFillNode, destoryFillPhysiNode ); - setFunc("PhysiSessionWindow", + setFunc("PhysiSessionWindow", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, sizeof(SSessionWinodwPhysiNode), physiSessionWindowNodeToJson, jsonToPhysiSessionWindowNode, destroyWindowPhysiNode ); - setFunc("PhysiStreamSessionWindow", + setFunc("PhysiStreamSessionWindow", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, sizeof(SStreamSessionWinodwPhysiNode), physiSessionWindowNodeToJson, jsonToPhysiSessionWindowNode, destroyWindowPhysiNode ); - setFunc("PhysiStreamSemiSessionWindow", + setFunc("PhysiStreamSemiSessionWindow", QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, sizeof(SStreamSemiSessionWinodwPhysiNode), physiSessionWindowNodeToJson, jsonToPhysiSessionWindowNode, destroyWindowPhysiNode ); - setFunc("PhysiStreamFinalSessionWindow", + setFunc("PhysiStreamFinalSessionWindow", QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, sizeof(SStreamFinalSessionWinodwPhysiNode), physiSessionWindowNodeToJson, jsonToPhysiSessionWindowNode, destroyWindowPhysiNode ); - setFunc("PhysiStateWindow", + setFunc("PhysiStateWindow", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, sizeof(SStateWinodwPhysiNode), physiStateWindowNodeToJson, jsonToPhysiStateWindowNode, destoryStateWindowPhysiNode ); - setFunc("PhysiStreamStateWindow", + setFunc("PhysiStreamStateWindow", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, sizeof(SStreamStateWinodwPhysiNode), physiStateWindowNodeToJson, jsonToPhysiStateWindowNode, destoryStateWindowPhysiNode ); - setFunc("PhysiPartition", + setFunc("PhysiPartition", QUERY_NODE_PHYSICAL_PLAN_PARTITION, sizeof(SPartitionPhysiNode), physiPartitionNodeToJson, jsonToPhysiPartitionNode, destroyPartitionPhysiNode ); - setFunc("PhysiStreamPartition", + setFunc("PhysiStreamPartition", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, sizeof(SStreamPartitionPhysiNode), physiStreamPartitionNodeToJson, jsonToPhysiStreamPartitionNode, destoryStreamPartitionPhysiNode ); - setFunc("PhysiIndefRowsFunc", + setFunc("PhysiIndefRowsFunc", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, sizeof(SIndefRowsFuncPhysiNode), physiIndefRowsFuncNodeToJson, jsonToPhysiIndefRowsFuncNode, destoryIndefRowsFuncPhysiNode ); - setFunc("PhysiInterpFunc", + setFunc("PhysiInterpFunc", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, sizeof(SInterpFuncLogicNode), physiInterpFuncNodeToJson, jsonToPhysiInterpFuncNode, destoryInterpFuncPhysiNode ); - setFunc("PhysiDispatch", + setFunc("PhysiDispatch", QUERY_NODE_PHYSICAL_PLAN_DISPATCH, sizeof(SDataDispatcherNode), physiDispatchNodeToJson, jsonToPhysiDispatchNode, destroyDataSinkNode ); - setFunc("PhysiInsert", + setFunc("PhysiInsert", QUERY_NODE_PHYSICAL_PLAN_INSERT, sizeof(SDataInserterNode), emptyNodeToJson, emptyJsonToNode, destoryDataInserterNode ); - setFunc("PhysiQueryInsert", + setFunc("PhysiQueryInsert", QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, sizeof(SQueryInserterNode), physiQueryInsertNodeToJson, jsonToPhysiQueryInsertNode, destoryQueryInserterNode ); - setFunc("PhysiDelete", + setFunc("PhysiDelete", QUERY_NODE_PHYSICAL_PLAN_DELETE, sizeof(SDataDeleterNode), physiDeleteNodeToJson, jsonToPhysiDeleteNode, destoryDataDeleterNode ); - setFunc("PhysiGroupCache", + setFunc("PhysiGroupCache", QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, sizeof(SGroupCachePhysiNode), physiGroupCacheNodeToJson, jsonToPhysiGroupCacheNode, destoryGroupCachePhysiNode ); - setFunc("PhysiDynamicQueryCtrl", + setFunc("PhysiDynamicQueryCtrl", QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, sizeof(SDynQueryCtrlPhysiNode), physiDynQueryCtrlNodeToJson, jsonToPhysiDynQueryCtrlNode, destoryDynQueryCtrlPhysiNode ); - setFunc("PhysiSubplan", + setFunc("PhysiSubplan", QUERY_NODE_PHYSICAL_SUBPLAN, sizeof(SSubplan), subplanToJson, jsonToSubplan, destorySubplanNode ); - setFunc("PhysiPlan", + setFunc("PhysiPlan", QUERY_NODE_PHYSICAL_PLAN, sizeof(SQueryPlan), planToJson, jsonToPlan, destoryPlanNode ); - setFunc("PhysiTableCountScan", + setFunc("PhysiTableCountScan", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, sizeof(STableCountScanPhysiNode), physiLastRowScanNodeToJson, jsonToPhysiScanNode, destoryLastRowScanPhysiNode ); - setFunc("PhysiMergeEventWindow", + setFunc("PhysiMergeEventWindow", QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, sizeof(SEventWinodwPhysiNode), physiEventWindowNodeToJson, jsonToPhysiEventWindowNode, destoryEventWindowPhysiNode ); - setFunc("PhysiStreamEventWindow", + setFunc("PhysiStreamEventWindow", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, sizeof(SStreamEventWinodwPhysiNode), physiEventWindowNodeToJson, jsonToPhysiEventWindowNode, destoryEventWindowPhysiNode ); - setFunc("PhysiHashJoin", + setFunc("PhysiHashJoin", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, sizeof(SHashJoinPhysiNode), physiHashJoinNodeToJson, From 084d0ab367b0e6c3730c9cff751e602fe3570fe1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 19 Nov 2023 23:56:34 +0800 Subject: [PATCH 04/43] fix destoryShowCreateTableStmt --- source/libs/nodes/src/nodesCodeFuncs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 981e2bbaca..6771677aac 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -7045,7 +7045,7 @@ void destoryShowCreateDatabaseStmt(SNode* pNode) { void destoryShowCreateTableStmt(SNode* pNode) { STableCfg* pCfg = (STableCfg*)(((SShowCreateTableStmt*)pNode)->pTableCfg); - taosMemoryFreeClear(pCfg); + taosMemoryFreeClear(((SShowCreateTableStmt*)pNode)->pDbCfg); if (NULL == pCfg) { return; } From 6e1e4702a5f80db2895bbd9bbf16ce76da6395ea Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 20 Nov 2023 11:13:09 +0800 Subject: [PATCH 05/43] adjust --- include/libs/nodes/nodes.h | 1 - source/dnode/mgmt/node_mgmt/src/dmEnv.c | 1 - source/libs/nodes/src/nodesCodeFuncs.c | 27 +++++++++++-------------- source/libs/nodes/src/nodesUtilFuncs.c | 1 - 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 136926a856..bcbfd245f8 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -101,7 +101,6 @@ typedef struct SNodeList { typedef struct SNodeAllocator SNodeAllocator; -void nodesInit(); int32_t nodesInitAllocatorSet(); void nodesDestroyAllocatorSet(); int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index c57043b8bb..acbb3554ce 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -156,7 +156,6 @@ extern void s3End(); int32_t dmInit() { dInfo("start to init dnode env"); - nodesInit(); if (dmDiskInit() != 0) return -1; if (!dmCheckDataDirVersion()) return -1; if (!dmCheckDiskSpace()) return -1; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 6771677aac..cf5bc528e3 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -46,7 +46,7 @@ typedef struct SBuiltinNodeDefinition { SBuiltinNodeDefinition funcNodes[QUERY_NODE_END] = {NULL}; static TdThreadOnce functionNodeInit = PTHREAD_ONCE_INIT; -static int32_t initNodeCode = -1; +volatile int32_t initNodeCode = -1; static void setFunc(const char* name, int32_t type, int32_t nodeSize, FExecNodeToJson toJsonFunc, FExecJsonToNode toNodeFunc, FExecDestoryNode destoryFunc) { @@ -59,19 +59,16 @@ static void setFunc(const char* name, int32_t type, int32_t nodeSize, FExecNodeT static void doInitNodeFuncArray(); -void nodesInit() { - taosThreadOnce(&functionNodeInit, doInitNodeFuncArray); -} - bool funcArrayCheck(int32_t type) { - if (type < 0 || QUERY_NODE_END < (type+1)) { - nodesError("funcArrayCheck unknown type = %d", type); + if (type < 0 || QUERY_NODE_END <= type) { + nodesError("funcArrayCheck out of range type = %d", type); return false; } if (initNodeCode != 0) { - nodesInit(); + taosThreadOnce(&functionNodeInit, doInitNodeFuncArray); } if (!funcNodes[type].name) { + nodesError("funcArrayCheck unsupported type = %d", type); return false; } return true; @@ -6494,7 +6491,7 @@ static int32_t jsonToInsertStmt(const SJson* pJson, void* pObj) { int32_t specificNodeToJson(const void* pObj, SJson* pJson) { ENodeType type = nodeType(pObj); if (!funcArrayCheck(type)) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_NOT_FOUND; } if (funcNodes[type].toJsonFunc) { @@ -6508,7 +6505,7 @@ int32_t specificNodeToJson(const void* pObj, SJson* pJson) { int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { ENodeType type = nodeType(pObj); if (!funcArrayCheck(type)) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_NOT_FOUND; } if (funcNodes[type].toNodeFunc) { @@ -7423,16 +7420,16 @@ void nodesDestroyNode(SNode* pNode) { return; } - int32_t index = nodeType(pNode); - if (!funcArrayCheck(index)) { + int32_t type = nodeType(pNode); + if (!funcArrayCheck(type)) { return; } - if (funcNodes[index].destoryFunc) { - funcNodes[index].destoryFunc(pNode); + if (funcNodes[type].destoryFunc) { + funcNodes[type].destoryFunc(pNode); nodesFree(pNode); return; } - nodesError("nodesDestroyNode unknown node type = %d", nodeType(pNode)); + nodesWarn("nodesDestroyNode unknown type = %d", type); nodesFree(pNode); return; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 3a738dbbd2..2ea387792b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -131,7 +131,6 @@ static void destroyNodeAllocator(void* p) { } int32_t nodesInitAllocatorSet() { - nodesInit(); if (g_allocatorReqRefPool >= 0) { nodesWarn("nodes already initialized"); return TSDB_CODE_SUCCESS; From ffc952e31eb36caf554700e46a36df3d24453dc7 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 20 Nov 2023 14:40:46 +0800 Subject: [PATCH 06/43] adjust funcNodes init --- source/libs/nodes/src/nodesCodeFuncs.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index cf5bc528e3..d4cd37750b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -48,6 +48,7 @@ SBuiltinNodeDefinition funcNodes[QUERY_NODE_END] = {NULL}; static TdThreadOnce functionNodeInit = PTHREAD_ONCE_INIT; volatile int32_t initNodeCode = -1; + static void setFunc(const char* name, int32_t type, int32_t nodeSize, FExecNodeToJson toJsonFunc, FExecJsonToNode toNodeFunc, FExecDestoryNode destoryFunc) { funcNodes[type].name = name; @@ -64,9 +65,9 @@ bool funcArrayCheck(int32_t type) { nodesError("funcArrayCheck out of range type = %d", type); return false; } - if (initNodeCode != 0) { - taosThreadOnce(&functionNodeInit, doInitNodeFuncArray); - } + // only init once, do nothing when initNodeCode == 0 + taosThreadOnce(&functionNodeInit, doInitNodeFuncArray); + if (!funcNodes[type].name) { nodesError("funcArrayCheck unsupported type = %d", type); return false; @@ -7436,6 +7437,10 @@ void nodesDestroyNode(SNode* pNode) { // clang-format off static void doInitNodeFuncArray() { + if (initNodeCode == 0) { + return; + } + setFunc("Column", QUERY_NODE_COLUMN, sizeof(SColumnNode), From c05a1c93334ff310601725d9f8cceed64d9b1e70 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 20 Nov 2023 15:25:58 +0800 Subject: [PATCH 07/43] test:update testcases in windows --- tests/develop-test/win-test-file | 3 ++ tests/script/win-test-file | 60 +++++------------------- tests/system-test/win-test-file | 79 +++++++++++++++++++++++++++++--- 3 files changed, 87 insertions(+), 55 deletions(-) diff --git a/tests/develop-test/win-test-file b/tests/develop-test/win-test-file index b640ef6bfe..1da890fc36 100644 --- a/tests/develop-test/win-test-file +++ b/tests/develop-test/win-test-file @@ -1,4 +1,7 @@ python3 ./test.py -f 2-query/table_count_scan.py +python3 ./test.py -f 2-query/pseudo_column.py +python3 ./test.py -f 2-query/ts-range.py +python3 ./test.py -f 2-query/tag_scan.py python3 ./test.py -f 2-query/show_create_db.py python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py diff --git a/tests/script/win-test-file b/tests/script/win-test-file index b2d50ade8a..d9ff09f468 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -19,19 +19,18 @@ ./test.sh -f tsim/dnode/balance3.sim ./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/stable/metrics_idx.sim -./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/sync/3Replica1VgElect.sim ./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/valgrind/checkError6.sim ./test.sh -f tsim/user/basic.sim ./test.sh -f tsim/user/password.sim +./test.sh -f tsim/user/whitelist.sim ./test.sh -f tsim/user/privilege_db.sim ./test.sh -f tsim/user/privilege_sysinfo.sim ./test.sh -f tsim/user/privilege_topic.sim ./test.sh -f tsim/user/privilege_table.sim ./test.sh -f tsim/user/privilege_create_db.sim ./test.sh -f tsim/db/alter_option.sim -./test.sh -f tsim/db/alter_replica_31.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim @@ -97,6 +96,7 @@ ./test.sh -f tsim/insert/delete0.sim ./test.sh -f tsim/insert/update1_sort_merge.sim ./test.sh -f tsim/insert/update2.sim +./test.sh -f tsim/insert/insert_stb.sim ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim @@ -186,6 +186,7 @@ ./test.sh -f tsim/query/session.sim ./test.sh -f tsim/query/join_interval.sim ./test.sh -f tsim/query/join_pk.sim +./test.sh -f tsim/query/count_spread.sim ./test.sh -f tsim/query/unionall_as_table.sim ./test.sh -f tsim/query/multi_order_by.sim ./test.sh -f tsim/query/sys_tbname.sim @@ -197,10 +198,14 @@ ./test.sh -f tsim/query/emptyTsRange_scl.sim ./test.sh -f tsim/query/partitionby.sim ./test.sh -f tsim/query/tableCount.sim +./test.sh -f tsim/query/show_db_table_kind.sim +./test.sh -f tsim/query/bi_star_table.sim +./test.sh -f tsim/query/bi_tag_scan.sim ./test.sh -f tsim/query/tag_scan.sim ./test.sh -f tsim/query/nullColSma.sim ./test.sh -f tsim/query/bug3398.sim ./test.sh -f tsim/query/explain_tsorder.sim +./test.sh -f tsim/query/apercentile.sim ./test.sh -f tsim/qnode/basic1.sim ./test.sh -f tsim/snode/basic1.sim ./test.sh -f tsim/mnode/basic1.sim @@ -237,52 +242,6 @@ ./test.sh -f tsim/table/table.sim ./test.sh -f tsim/table/tinyint.sim ./test.sh -f tsim/table/vgroup.sim -./test.sh -f tsim/stream/basic0.sim -g -./test.sh -f tsim/stream/basic1.sim -./test.sh -f tsim/stream/basic2.sim -./test.sh -f tsim/stream/basic3.sim -./test.sh -f tsim/stream/basic4.sim -./test.sh -f tsim/stream/checkpointInterval0.sim -./test.sh -f tsim/stream/checkStreamSTable1.sim -./test.sh -f tsim/stream/checkStreamSTable.sim -./test.sh -f tsim/stream/deleteInterval.sim -./test.sh -f tsim/stream/deleteSession.sim -./test.sh -f tsim/stream/deleteState.sim -./test.sh -f tsim/stream/distributeInterval0.sim -./test.sh -f tsim/stream/distributeIntervalRetrive0.sim -./test.sh -f tsim/stream/distributeSession0.sim -./test.sh -f tsim/stream/drop_stream.sim -./test.sh -f tsim/stream/fillHistoryBasic1.sim -./test.sh -f tsim/stream/fillHistoryBasic2.sim -./test.sh -f tsim/stream/fillHistoryBasic3.sim -./test.sh -f tsim/stream/fillIntervalDelete0.sim -./test.sh -f tsim/stream/fillIntervalDelete1.sim -./test.sh -f tsim/stream/fillIntervalLinear.sim -./test.sh -f tsim/stream/fillIntervalPartitionBy.sim -./test.sh -f tsim/stream/fillIntervalPrevNext1.sim -./test.sh -f tsim/stream/fillIntervalPrevNext.sim -./test.sh -f tsim/stream/fillIntervalRange.sim -./test.sh -f tsim/stream/fillIntervalValue.sim -./test.sh -f tsim/stream/ignoreCheckUpdate.sim -./test.sh -f tsim/stream/ignoreExpiredData.sim -./test.sh -f tsim/stream/partitionby1.sim -./test.sh -f tsim/stream/partitionbyColumnInterval.sim -./test.sh -f tsim/stream/partitionbyColumnSession.sim -./test.sh -f tsim/stream/partitionbyColumnState.sim -./test.sh -f tsim/stream/partitionby.sim -./test.sh -f tsim/stream/pauseAndResume.sim -./test.sh -f tsim/stream/schedSnode.sim -./test.sh -f tsim/stream/session0.sim -./test.sh -f tsim/stream/session1.sim -./test.sh -f tsim/stream/sliding.sim -./test.sh -f tsim/stream/state0.sim -./test.sh -f tsim/stream/state1.sim -./test.sh -f tsim/stream/triggerInterval0.sim -./test.sh -f tsim/stream/triggerSession0.sim -./test.sh -f tsim/stream/udTableAndTag0.sim -./test.sh -f tsim/stream/udTableAndTag1.sim -./test.sh -f tsim/stream/udTableAndTag2.sim -./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/trans/lossdata1.sim ./test.sh -f tsim/tmq/basic1.sim ./test.sh -f tsim/tmq/basic2.sim @@ -431,3 +390,8 @@ ./test.sh -f tsim/tag/drop_tag.sim ./test.sh -f tsim/tag/tbNameIn.sim ./test.sh -f tmp/monitor.sim +./test.sh -f tsim/tagindex/add_index.sim +./test.sh -f tsim/tagindex/sma_and_tag_index.sim +./test.sh -f tsim/view/view.sim +./test.sh -f tsim/query/cache_last.sim +./test.sh -f tsim/query/const.sim diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index 3daf65b406..aefdb1e824 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -1,3 +1,12 @@ +python3 ./test.py -f 2-query/tbname_vgroup.py +python3 ./test.py -f 2-query/stbJoin.py +python3 ./test.py -f 2-query/stbJoin.py -Q 2 +python3 ./test.py -f 2-query/stbJoin.py -Q 3 +python3 ./test.py -f 2-query/stbJoin.py -Q 4 +python3 ./test.py -f 2-query/hint.py +python3 ./test.py -f 2-query/hint.py -Q 2 +python3 ./test.py -f 2-query/hint.py -Q 3 +python3 ./test.py -f 2-query/hint.py -Q 4 python3 ./test.py -f 2-query/nestedQuery.py python3 ./test.py -f 2-query/nestedQuery_str.py python3 ./test.py -f 2-query/nestedQuery_math.py @@ -18,7 +27,30 @@ python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4 +python3 ./test.py -f 2-query/interval_unit.py +python3 ./test.py -f 2-query/interval_unit.py -Q 2 +python3 ./test.py -f 2-query/interval_unit.py -Q 3 +python3 ./test.py -f 2-query/interval_unit.py -Q 4 python3 ./test.py -f 2-query/partition_by_col.py -Q 4 +python3 ./test.py -f 2-query/partition_by_col.py -Q 3 +python3 ./test.py -f 2-query/partition_by_col.py -Q 2 +python3 ./test.py -f 2-query/partition_by_col.py +python3 ./test.py -f 2-query/partition_by_col_agg.py +python3 ./test.py -f 2-query/partition_by_col_agg.py -Q 2 +python3 ./test.py -f 2-query/partition_by_col_agg.py -Q 3 +python3 ./test.py -f 2-query/partition_by_col_agg.py -Q 4 +python3 ./test.py -f 2-query/interval_limit_opt_2.py -Q 4 +python3 ./test.py -f 2-query/interval_limit_opt_2.py -Q 3 +python3 ./test.py -f 2-query/interval_limit_opt_2.py -Q 2 +python3 ./test.py -f 2-query/interval_limit_opt_2.py +python3 ./test.py -f 2-query/func_to_char_timestamp.py +python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 2 +python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 3 +python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 4 +python3 ./test.py -f 2-query/last_cache_scan.py +python3 ./test.py -f 2-query/last_cache_scan.py -Q 2 +python3 ./test.py -f 2-query/last_cache_scan.py -Q 3 +python3 ./test.py -f 2-query/last_cache_scan.py -Q 4 python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqDropStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -29,11 +61,13 @@ python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 python3 ./test.py -f 7-tmq/ins_topics_test.py python3 ./test.py -f 7-tmq/tmqMaxTopic.py python3 ./test.py -f 7-tmq/tmqParamsTest.py +python3 ./test.py -f 7-tmq/tmqParamsTest.py -R python3 ./test.py -f 7-tmq/tmqClientConsLog.py python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py python3 ./test.py -f 7-tmq/tmqOffset.py python3 ./test.py -f 7-tmq/tmqDropConsumer.py +python3 ./test.py -f 1-insert/insert_stb.py python3 ./test.py -f 1-insert/delete_stable.py python3 ./test.py -f 2-query/out_of_order.py -Q 3 python3 ./test.py -f 2-query/out_of_order.py @@ -61,10 +95,14 @@ python3 ./test.py -f 2-query/slimit.py -R python3 ./test.py -f 2-query/slimit.py -Q 2 python3 ./test.py -f 2-query/slimit.py -Q 3 python3 ./test.py -f 2-query/slimit.py -Q 4 -python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3 +python3 ./test.py -f 2-query/ts-4233.py +python3 ./test.py -f 2-query/ts-4233.py -Q 2 +python3 ./test.py -f 2-query/ts-4233.py -Q 3 +python3 ./test.py -f 2-query/ts-4233.py -Q 4 +python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False +python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False +python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -i False +python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3 -i False python3 ./test.py -f 7-tmq/create_wrong_topic.py python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py @@ -112,16 +150,30 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py python3 ./test.py -f 7-tmq/tmq_taosx.py +python3 ./test.py -f 7-tmq/tmq_replay.py +python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py +python3 ./test.py -f 7-tmq/tmq_offset.py +python3 ./test.py -f 7-tmq/tmqDataPrecisionUnit.py python3 ./test.py -f 7-tmq/raw_block_interface_test.py python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True +python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 +python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 python3 ./test.py -f 99-TDcase/TD-19201.py python3 ./test.py -f 99-TDcase/TD-21561.py python3 ./test.py -f 99-TDcase/TS-3404.py python3 ./test.py -f 99-TDcase/TS-3581.py python3 ./test.py -f 99-TDcase/TS-3311.py +python3 ./test.py -f 99-TDcase/TS-3821.py python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py @@ -139,10 +191,19 @@ python3 ./test.py -f 0-others/user_privilege_show.py python3 ./test.py -f 0-others/user_privilege_all.py python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/multilevel.py +python3 ./test.py -f 0-others/ttl.py +python3 ./test.py -f 0-others/ttlChangeOnWrite.py +python3 ./test.py -f 0-others/compress_tsz1.py +python3 ./test.py -f 0-others/compress_tsz2.py +python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py +python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 +python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 +python3 ./test.py -f 0-others/delete_check.py +python3 ./test.py -f 0-others/test_hot_refresh_configurations.py python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/alter_replica.py -N 3 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py @@ -190,6 +251,7 @@ python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 3 python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 python3 ./test.py -f 1-insert/precisionUS.py python3 ./test.py -f 1-insert/precisionNS.py +python3 ./test.py -f 1-insert/test_ts4219.py python3 ./test.py -f 0-others/show.py python3 ./test.py -f 0-others/show_tag_index.py python3 ./test.py -f 0-others/information_schema.py @@ -325,6 +387,8 @@ python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py -R python3 ./test.py -f 0-others/sma_index.py python3 ./test.py -f 2-query/sml_TS-3724.py +python3 ./test.py -f 2-query/sml-TD19291.py +python3 ./test.py -f 2-query/varbinary.py python3 ./test.py -f 2-query/sml.py python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/spread.py @@ -426,7 +490,6 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 6 -M 3 @@ -731,7 +794,9 @@ python3 ./test.py -f 2-query/out_of_order.py -R python3 ./test.py -f 2-query/blockSMA.py -Q 4 python3 ./test.py -f 2-query/projectionDesc.py -Q 4 python3 ./test.py -f 2-query/odbc.py +python3 ./test.py -f 2-query/fill_with_group.py python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 python3 ./test.py -f 99-TDcase/TD-20582.py python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 +python3 ./test.py -f eco-system/meta/database/keep_time_offset.py From 146c36b0ad168bc93a4d86d9eb569b701e662ecd Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 20 Nov 2023 19:25:31 +0800 Subject: [PATCH 08/43] rm unused --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 1 - source/libs/nodes/src/nodesCodeFuncs.c | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index acbb3554ce..6f13abcebc 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -17,7 +17,6 @@ #include "dmMgmt.h" #include "audit.h" #include "libs/function/tudf.h" -#include "nodes.h" #define DM_INIT_AUDIT() \ do { \ diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d4cd37750b..0a4f9ccdd5 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -46,7 +46,6 @@ typedef struct SBuiltinNodeDefinition { SBuiltinNodeDefinition funcNodes[QUERY_NODE_END] = {NULL}; static TdThreadOnce functionNodeInit = PTHREAD_ONCE_INIT; -volatile int32_t initNodeCode = -1; static void setFunc(const char* name, int32_t type, int32_t nodeSize, FExecNodeToJson toJsonFunc, @@ -65,7 +64,7 @@ bool funcArrayCheck(int32_t type) { nodesError("funcArrayCheck out of range type = %d", type); return false; } - // only init once, do nothing when initNodeCode == 0 + // only init once, do nothing when run funcArrayCheck again taosThreadOnce(&functionNodeInit, doInitNodeFuncArray); if (!funcNodes[type].name) { @@ -7437,10 +7436,6 @@ void nodesDestroyNode(SNode* pNode) { // clang-format off static void doInitNodeFuncArray() { - if (initNodeCode == 0) { - return; - } - setFunc("Column", QUERY_NODE_COLUMN, sizeof(SColumnNode), @@ -8848,7 +8843,6 @@ static void doInitNodeFuncArray() { jsonToPhysiHashJoinNode, destoryHashJoinPhysiNode ); - initNodeCode = 0; } // clang-format on From 9a09e318ca91d58096e2d1ddfa5f16171ba3ac9e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 22 Nov 2023 16:53:33 +0800 Subject: [PATCH 09/43] opti:common message process logic in vnode and snode --- include/dnode/vnode/stream.h | 18 - include/libs/stream/tstream.h | 21 + source/dnode/mgmt/mgmt_snode/src/smWorker.c | 6 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 6 +- source/dnode/snode/CMakeLists.txt | 1 - source/dnode/snode/src/snode.c | 780 +------------------ source/dnode/vnode/CMakeLists.txt | 5 - source/dnode/vnode/inc/vnode.h | 2 - source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 3 +- source/dnode/vnode/src/tq/tq.c | 570 +------------- source/dnode/vnode/src/tq/tqStreamTask.c | 170 ----- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/dnode/vnode/src/vnd/vnodeSync.c | 4 +- source/libs/stream/inc/streamInt.h | 6 + source/libs/stream/src/stream.c | 1 + source/libs/stream/src/streamStart.c | 4 +- source/libs/stream/src/streamTask.c | 785 +++++++++++++++++++- 18 files changed, 864 insertions(+), 1522 deletions(-) delete mode 100644 include/dnode/vnode/stream.h diff --git a/include/dnode/vnode/stream.h b/include/dnode/vnode/stream.h deleted file mode 100644 index 6d86847542..0000000000 --- a/include/dnode/vnode/stream.h +++ /dev/null @@ -1,18 +0,0 @@ -// -// Created by mingming wanng on 2023/11/15. -// - -#ifndef TDENGINE_STREAM_H -#define TDENGINE_STREAM_H - -#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) -#define STREAM_EXEC_START_ALL_TASKS_ID (-2) -#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) - -typedef struct STaskUpdateEntry { - int64_t streamId; - int32_t taskId; - int32_t transId; -} STaskUpdateEntry; - -#endif // TDENGINE_STREAM_H diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 159a842c44..69a44473f0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -51,6 +51,10 @@ extern "C" { (_t)->hTaskInfo.id.streamId = 0; \ } while (0) +#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) +#define STREAM_EXEC_START_ALL_TASKS_ID (-2) +#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) + typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; typedef struct SStreamTaskSM SStreamTaskSM; @@ -849,6 +853,23 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int8_t isSucceed); +// message process +int32_t streamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart); +int32_t streamTaskProcesUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored); +int32_t streamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); +int32_t streamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t streamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); +int32_t streamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); +int32_t streamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); +int32_t startStreamTasks(SStreamMeta* pMeta); +int32_t resetStreamTaskStatus(SStreamMeta* pMeta); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index e8402eb7c0..9220d3395d 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -193,10 +193,6 @@ int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->streamWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); - if (pMsg->msgType == TDMT_STREAM_TASK_DISPATCH) { - sndEnqueueStreamDispatch(pMgmt->pSnode, pMsg); - } else { - taosWriteQitem(pWorker->queue, pMsg); - } + taosWriteQitem(pWorker->queue, pMsg); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 4b18ec4fb0..b528913a23 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -206,11 +206,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); - if (pMsg->msgType == TDMT_STREAM_TASK_DISPATCH) { - vnodeEnqueueStreamMsg(pVnode->pImpl, pMsg); - } else { - taosWriteQitem(pVnode->pStreamQ, pMsg); - } + taosWriteQitem(pVnode->pStreamQ, pMsg); break; case FETCH_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 2da1f9adac..ebfe80ecab 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -3,7 +3,6 @@ add_library(snode STATIC ${SNODE_SRC}) target_include_directories( snode PUBLIC "${TD_SOURCE_DIR}/include/dnode/snode" - PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 007243192c..c0fbd2b2c3 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -18,7 +18,6 @@ #include "sndInt.h" #include "tstream.h" #include "tuuid.h" -#include "stream.h" #define sndError(...) \ do { \ @@ -41,45 +40,6 @@ } \ } while (0) -void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { - char *msgStr = pMsg->pCont; - char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; - - SStreamDispatchReq req; - SDecoder decoder; - tDecoderInit(&decoder, msgBody, msgLen); - if (tDecodeStreamDispatchReq(&decoder, &req) < 0) { - code = TSDB_CODE_MSG_DECODE_ERROR; - tDecoderClear(&decoder); - goto FAIL; - } - - tDecoderClear(&decoder); - - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.taskId); - if (pTask) { - SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; - streamProcessDispatchMsg(pTask, &req, &rsp); - tDeleteStreamDispatchReq(&req); - streamMetaReleaseTask(pSnode->pMeta, pTask); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - return; - } else { - tDeleteStreamDispatchReq(&req); - return; - } - -FAIL: - if (pMsg->info.handle == NULL) return; - SRpcMsg rsp = { .code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0); int32_t code = streamTaskInit(pTask, pSnode->pMeta, &pSnode->msgCb, nextProcessVer); @@ -130,159 +90,29 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); - } else { - if (pTask->chkInfo.nextProcessVer == -1) { - pTask->chkInfo.nextProcessVer = 0; - } } char* p = NULL; streamTaskGetStatus(pTask, &p); - sndInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", - SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, - pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam); - + if (pTask->info.fillHistory) { + sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " nextProcessVer:%" PRId64 + " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, + pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, + (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); + } else { + sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " nextProcessVer:%" PRId64 + " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, + pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, + (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); + } return 0; } -int32_t sndStartStreamTasks(SSnode* pSnode) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = SNODE_HANDLE; - SStreamMeta* pMeta = pSnode->pMeta; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - sndDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - taosHashClear(pMeta->startInfo.pReadyTaskSet); - taosHashClear(pMeta->startInfo.pFailedTaskSet); - pMeta->startInfo.startTs = taosGetTimestampMs(); - streamMetaWUnLock(pMeta); - - // broadcast the check downstream tasks msg - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - continue; - } - - // fill-history task can only be launched by related stream tasks. - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - if (pTask->status.downstreamReady == 1) { - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - sndDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); - } - - streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); - if (ret != TSDB_CODE_SUCCESS) { - code = ret; - } - - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - return code; -} - -int32_t sndResetStreamTaskStatus(SSnode* pSnode) { - SStreamMeta* pMeta = pSnode->pMeta; - int32_t vgId = pMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - - sndDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); - - STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; - SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - streamTaskResetStatus(*pTask); - } - - return 0; -} - -int32_t sndRestartStreamTasks(SSnode* pSnode) { - SStreamMeta* pMeta = pSnode->pMeta; - int32_t vgId = pMeta->vgId; - int32_t code = 0; - int64_t st = taosGetTimestampMs(); - - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - sndDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - terrno = 0; - sndInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, - pMeta->updateInfo.transId); - - while (streamMetaTaskInTimer(pMeta)) { - sndDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - code = streamMetaReopen(pMeta); - if (code != TSDB_CODE_SUCCESS) { - sndError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - code = terrno; - return code; - } - - streamMetaInitBackend(pMeta); - int64_t el = taosGetTimestampMs() - st; - - sndInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); - - code = streamMetaLoadAllTasks(pMeta); - if (code != TSDB_CODE_SUCCESS) { - sndError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); - streamMetaWUnLock(pMeta); - code = terrno; - return code; - } - sndInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); - sndResetStreamTaskStatus(pSnode); - - streamMetaWUnLock(pMeta); - sndStartStreamTasks(pSnode); - - code = terrno; - return code; -} - SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); if (pSnode == NULL) { @@ -318,8 +148,8 @@ FAIL: } int32_t sndInit(SSnode * pSnode) { - sndResetStreamTaskStatus(pSnode); - sndStartStreamTasks(pSnode); + resetStreamTaskStatus(pSnode->pMeta); + startStreamTasks(pSnode->pMeta); return 0; } @@ -331,577 +161,26 @@ void sndClose(SSnode *pSnode) { taosMemoryFree(pSnode); } -int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } - -int32_t sndStartStreamTaskAsync(SSnode* pSnode, bool restart) { - SStreamMeta* pMeta = pSnode->pMeta; - int32_t vgId = pMeta->vgId; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - sndDebug("vgId:%d no stream tasks existed to run", vgId); - return 0; - } - - SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - sndError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - return -1; - } - - sndDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); - pRunReq->head.vgId = vgId; - pRunReq->streamId = 0; - pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - tmsgPutToQueue(&pSnode->msgCb, STREAM_QUEUE, &msg); - return 0; -} - -int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { - int32_t code; - - // 1.deserialize msg and build task - SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - return -1; - } - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t *)msg, msgLen); - code = tDecodeStreamTask(&decoder, pTask); - if (code < 0) { - tDecoderClear(&decoder); - taosMemoryFree(pTask); - return -1; - } - - tDecoderClear(&decoder); - - ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG); - - // 2.save task - streamMetaWLock(pSnode->pMeta); - - bool added = false; - code = streamMetaRegisterTask(pSnode->pMeta, -1, pTask, &added); - if (code < 0) { - streamMetaWUnLock(pSnode->pMeta); - return -1; - } - - int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); - streamMetaWUnLock(pSnode->pMeta); - - char* p = NULL; - streamTaskGetStatus(pTask, &p); - - sndDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, - pTask->id.idStr, p, numOfTasks); - - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - streamTaskHandleEvent(pTask->status.pSM, event); - return 0; -} - -int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { - SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; - sndDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); - streamMetaUnregisterTask(pSnode->pMeta, pReq->streamId, pReq->taskId); - - // commit the update - streamMetaWLock(pSnode->pMeta); - int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); - sndDebug("vgId:%d task:0x%x dropped, remain tasks:%d", pSnode->pMeta->vgId, pReq->taskId, numOfTasks); - - if (streamMetaCommit(pSnode->pMeta) < 0) { - // persist to disk - } - streamMetaWUnLock(pSnode->pMeta); - return 0; -} - -int32_t sndProcessTaskRunReq(SSnode *pSnode, SRpcMsg *pMsg) { - SStreamTaskRunReq *pReq = pMsg->pCont; - - int32_t taskId = pReq->taskId; - - if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { - sndStartStreamTasks(pSnode); - return 0; - } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { - sndRestartStreamTasks(pSnode); - return 0; - } - - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); - if (pTask) { - streamExecTask(pTask); - streamMetaReleaseTask(pSnode->pMeta, pTask); - return 0; - } else { - return -1; - } -} - -int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) { - char *msgStr = pMsg->pCont; - char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - SStreamDispatchReq req; - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); - tDecodeStreamDispatchReq(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.taskId); - if (pTask) { - SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessDispatchMsg(pTask, &req, &rsp); - tDeleteStreamDispatchReq(&req); - streamMetaReleaseTask(pSnode->pMeta, pTask); - return 0; - } else { - tDeleteStreamDispatchReq(&req); - return -1; - } -} - -int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) { - char *msgStr = pMsg->pCont; - char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - SStreamRetrieveReq req; - SDecoder decoder; - tDecoderInit(&decoder, msgBody, msgLen); - tDecodeStreamRetrieveReq(&decoder, &req); - tDecoderClear(&decoder); - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.dstTaskId); - - if (pTask) { - SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessRetrieveReq(pTask, &req, &rsp); - streamMetaReleaseTask(pSnode->pMeta, pTask); - tDeleteStreamRetrieveReq(&req); - return 0; - } else { - return -1; - } -} - -int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { - SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - - pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); - pRsp->streamId = htobe64(pRsp->streamId); - pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); - pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); - pRsp->stage = htobe64(pRsp->stage); - pRsp->msgId = htonl(pRsp->msgId); - - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pRsp->streamId, pRsp->upstreamTaskId); - if (pTask) { - streamProcessDispatchRsp(pTask, pRsp, pMsg->code); - streamMetaReleaseTask(pSnode->pMeta, pTask); - return 0; - } else { - return -1; - } -} - -int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) { - // - return 0; -} - -int32_t sndProcessTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { - char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - // deserialize - SStreamScanHistoryFinishReq req; - - SDecoder decoder; - tDecoderInit(&decoder, msg, msgLen); - tDecodeStreamScanHistoryFinishReq(&decoder, &req); - tDecoderClear(&decoder); - - // find task - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.downstreamTaskId); - if (pTask == NULL) { - return -1; - } - // do process request - if (streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info) < 0) { - streamMetaReleaseTask(pSnode->pMeta, pTask); - return -1; - } - - streamMetaReleaseTask(pSnode->pMeta, pTask); - return 0; -} - -int32_t sndProcessTaskScanHistoryFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - // deserialize - SStreamCompleteHistoryMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeCompleteHistoryDataMsg(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.upstreamTaskId); - if (pTask == NULL) { - sndError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", - pSnode->pMeta->vgId, req.upstreamTaskId); - return -1; - } - - int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); - if (remain > 0) { - sndDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", - pTask->id.idStr, req.downstreamId, remain); - } else { - sndDebug( - "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " - "completed msg", - pTask->id.idStr, req.downstreamId); - streamProcessScanHistoryFinishRsp(pTask); - } - - streamMetaReleaseTask(pSnode->pMeta, pTask); - return 0; -} - -// downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task -int32_t sndProcessTaskCheckpointReadyMsg(SSnode *pSnode, SRpcMsg* pMsg) { - SStreamMeta* pMeta = pSnode->pMeta; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; - - SStreamCheckpointReadyMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { - code = TSDB_CODE_MSG_DECODE_ERROR; - tDecoderClear(&decoder); - return code; - } - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); - if (pTask == NULL) { - sndError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); - return code; - } - - sndDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, - pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); - - streamProcessCheckpointReadyMsg(pTask); - streamMetaReleaseTask(pMeta, pTask); - return code; -} - -int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { - char *msgStr = pMsg->pCont; - char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - SStreamTaskCheckReq req; - SDecoder decoder; - - tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); - tDecodeStreamTaskCheckReq(&decoder, &req); - tDecoderClear(&decoder); - - int32_t taskId = req.downstreamTaskId; - - SStreamTaskCheckRsp rsp = { - .reqId = req.reqId, - .streamId = req.streamId, - .childId = req.childId, - .downstreamNodeId = req.downstreamNodeId, - .downstreamTaskId = req.downstreamTaskId, - .upstreamNodeId = req.upstreamNodeId, - .upstreamTaskId = req.upstreamTaskId, - }; - - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, taskId); - - if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); - streamMetaReleaseTask(pSnode->pMeta, pTask); - char* p = NULL; - streamTaskGetStatus(pTask, &p); - sndDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", - pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - } else { - rsp.status = TASK_DOWNSTREAM_NOT_READY; - sndDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", - taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - } - - SEncoder encoder; - int32_t code; - int32_t len; - - tEncodeSize(tEncodeStreamTaskCheckRsp, &rsp, len, code); - if (code < 0) { - sndError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); - return -1; - } - - void *buf = rpcMallocCont(sizeof(SMsgHead) + len); - ((SMsgHead *)buf)->vgId = htonl(req.upstreamNodeId); - - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncoderInit(&encoder, (uint8_t *)abuf, len); - tEncodeStreamTaskCheckRsp(&encoder, &rsp); - tEncoderClear(&encoder); - - SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info}; - - tmsgSendRsp(&rspMsg); - return 0; -} - -int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { - char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - - int32_t code; - SStreamTaskCheckRsp rsp; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)pReq, len); - code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); - - if (code < 0) { - tDecoderClear(&decoder); - return -1; - } - - tDecoderClear(&decoder); - sndDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", - rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); - - SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.streamId, rsp.upstreamTaskId); - if (pTask == NULL) { - sndError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, - pSnode->pMeta->vgId); - return -1; - } - - code = streamProcessCheckRsp(pTask, &rsp); - streamMetaReleaseTask(pSnode->pMeta, pTask); - return code; -} - -int32_t sndProcessTaskUpdateReq(SSnode* pSnode, SRpcMsg* pMsg) { - SStreamMeta* pMeta = pSnode->pMeta; - int32_t vgId = SNODE_HANDLE; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; - - SStreamTaskNodeUpdateMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { - rsp.code = TSDB_CODE_MSG_DECODE_ERROR; - sndError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); - tDecoderClear(&decoder); - return rsp.code; - } - - tDecoderClear(&decoder); - - // update the nodeEpset when it exists - streamMetaWLock(pMeta); - - // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. - STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - if (ppTask == NULL || *ppTask == NULL) { - sndError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, - req.taskId); - rsp.code = TSDB_CODE_SUCCESS; - streamMetaWUnLock(pMeta); - - taosArrayDestroy(req.pNodeList); - return rsp.code; - } - - SStreamTask* pTask = *ppTask; - - if (pMeta->updateInfo.transId != req.transId) { - pMeta->updateInfo.transId = req.transId; - sndInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); - // info needs to be kept till the new trans to update the nodeEp arrived. - taosHashClear(pMeta->updateInfo.pTasks); - } else { - sndDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); - } - - STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; - void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); - if (exist != NULL) { - sndDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, - req.transId); - rsp.code = TSDB_CODE_SUCCESS; - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return rsp.code; - } - - streamMetaWUnLock(pMeta); - - // the following two functions should not be executed within the scope of meta lock to avoid deadlock - streamTaskUpdateEpsetInfo(pTask, req.pNodeList); - streamTaskResetStatus(pTask); - - // continue after lock the meta again - streamMetaWLock(pMeta); - - SStreamTask** ppHTask = NULL; - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); - if (ppHTask == NULL || *ppHTask == NULL) { - sndError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - pMeta->vgId, req.taskId); - CLEAR_RELATED_FILLHISTORY_TASK(pTask); - } else { - sndDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); - streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); - } - } - - { - streamMetaSaveTask(pMeta, pTask); - if (ppHTask != NULL) { - streamMetaSaveTask(pMeta, *ppHTask); - } - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } - } - - streamTaskStop(pTask); - - // keep the already handled info - taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); - - if (ppHTask != NULL) { - streamTaskStop(*ppHTask); - sndDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); - taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); - } else { - sndDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); - } - - rsp.code = 0; - - // possibly only handle the stream task. - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); - - pMeta->startInfo.tasksWillRestart = 1; - - if (updateTasks < numOfTasks) { - sndDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, - updateTasks, (numOfTasks - updateTasks)); - streamMetaWUnLock(pMeta); - } else { - sndDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); -#if 1 - sndStartStreamTaskAsync(pSnode, true); - streamMetaWUnLock(pMeta); -#else - streamMetaWUnLock(pMeta); - - // For debug purpose. - // the following procedure consume many CPU resource, result in the re-election of leader - // with high probability. So we employ it as a test case for the stream processing framework, with - // checkpoint/restart/nodeUpdate etc. - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - int32_t code = streamMetaReopen(pMeta); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { - tqError("vgId:%d failed to load stream tasks", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - tqResetStreamTaskStatus(pTq); - tqStartStreamTaskAsync(pTq, false); - } else { - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - streamMetaWUnLock(pMeta); -#endif - } - - taosArrayDestroy(req.pNodeList); - return rsp.code; -} - int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: - return sndProcessTaskRunReq(pSnode, pMsg); + return streamTaskProcessRunReq(pSnode->pMeta, pMsg, true); case TDMT_STREAM_TASK_DISPATCH: - return sndProcessTaskDispatchReq(pSnode, pMsg, true); + return streamTaskProcessDispatchReq(pSnode->pMeta, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: - return sndProcessTaskDispatchRsp(pSnode, pMsg); + return streamTaskProcessDispatchRsp(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE: - return sndProcessTaskRetrieveReq(pSnode, pMsg); - case TDMT_STREAM_RETRIEVE_RSP: - return sndProcessTaskRetrieveRsp(pSnode, pMsg); + return streamTaskProcessRetrieveReq(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH: - return sndProcessTaskScanHistoryFinishReq(pSnode, pMsg); + return streamTaskProcessScanHistoryFinishReq(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP: - return sndProcessTaskScanHistoryFinishRsp(pSnode, pMsg); + return streamTaskProcessScanHistoryFinishRsp(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_TASK_CHECK: - return sndProcessStreamTaskCheckReq(pSnode, pMsg); + return streamTaskProcessCheckReq(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: - return sndProcessStreamTaskCheckRsp(pSnode, pMsg); + return streamTaskProcessCheckRsp(pSnode->pMeta, pMsg, true); case TDMT_STREAM_TASK_CHECKPOINT_READY: - return sndProcessTaskCheckpointReadyMsg(pSnode, pMsg); + return streamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg); default: ASSERT(0); } @@ -913,14 +192,13 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DEPLOY: { void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return sndProcessTaskDeployReq(pSnode, pReq, len); + return streamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); } case TDMT_STREAM_TASK_DROP: - return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); + return streamTaskProcessDropReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); case TDMT_VND_STREAM_TASK_UPDATE: - sndProcessTaskUpdateReq(pSnode, pMsg); - break; + return streamTaskProcesUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); default: ASSERT(0); } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 635c15aa41..dc43da7fe7 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -138,11 +138,6 @@ else() endif() endif() -target_include_directories( - vnode - PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" -) - target_link_libraries( vnode PUBLIC os diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6a0c991be4..9307a620ad 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -253,8 +253,6 @@ bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); -int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); - // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index b3f8317add..cf57623a43 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -153,7 +153,6 @@ char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream -int32_t tqResetStreamTaskStatus(STQ* pTq); int32_t tqStopStreamTasks(STQ* pTq); // tq util diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index df1720d4a7..a14e91d7e6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -233,7 +233,6 @@ int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart); int32_t tqRestartStreamTasks(STQ* pTq); int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqScanWal(STQ* pTq); @@ -262,7 +261,7 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t version, char* msg, int32_t msg int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); +int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 37507c7f4f..5daef7d5db 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,7 +15,6 @@ #include "tq.h" #include "vnd.h" -#include "stream.h" typedef struct { int8_t inited; @@ -891,172 +890,15 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { } int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - SStreamMeta* pMeta = pTq->pStreamMeta; - - SStreamTaskCheckReq req; - SDecoder decoder; - - tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamTaskCheckReq(&decoder, &req); - tDecoderClear(&decoder); - - int32_t taskId = req.downstreamTaskId; - - SStreamTaskCheckRsp rsp = { - .reqId = req.reqId, - .streamId = req.streamId, - .childId = req.childId, - .downstreamNodeId = req.downstreamNodeId, - .downstreamTaskId = req.downstreamTaskId, - .upstreamNodeId = req.upstreamNodeId, - .upstreamTaskId = req.upstreamTaskId, - }; - - // only the leader node handle the check request - if (pMeta->role == NODE_ROLE_FOLLOWER) { - tqError("s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", - taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); - rsp.status = TASK_DOWNSTREAM_NOT_LEADER; - } else { - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); - if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); - streamMetaReleaseTask(pMeta, pTask); - - char* p = NULL; - streamTaskGetStatus(pTask, &p); - tqDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", - pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - } else { - rsp.status = TASK_DOWNSTREAM_NOT_READY; - tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 - ") from task:0x%x (vgId:%d), rsp check_status %d", - req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - } - } - - return streamSendCheckRsp(pMeta, &req, &rsp, &pMsg->info, taskId); + return streamTaskProcessCheckReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) { - char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t vgId = pTq->pStreamMeta->vgId; - - int32_t code; - SStreamTaskCheckRsp rsp; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)pReq, len); - code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); - if (code < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tDecoderClear(&decoder); - tqError("vgId:%d failed to parse check rsp msg, code:%s", vgId, tstrerror(terrno)); - return -1; - } - - tDecoderClear(&decoder); - tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", - rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); - - if (!vnodeIsRoleLeader(pTq->pVnode)) { - tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, - rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); - return code; - } - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId); - if (pTask == NULL) { - tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", - rsp.streamId, rsp.upstreamTaskId, pTq->pStreamMeta->vgId); - terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; - return -1; - } - - code = streamProcessCheckRsp(pTask, &rsp); - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return code; + return streamTaskProcessCheckRsp(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); } int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - int32_t code = 0; - int32_t vgId = TD_VID(pTq->pVnode); - - if (tsDisableStream) { - tqInfo("vgId:%d stream disabled, not deploy stream tasks", vgId); - return code; - } - - tqDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId); - - // 1.deserialize msg and build task - int32_t size = sizeof(SStreamTask); - SStreamTask* pTask = taosMemoryCalloc(1, size); - if (pTask == NULL) { - tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size); - return TSDB_CODE_OUT_OF_MEMORY; - } - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeStreamTask(&decoder, pTask); - tDecoderClear(&decoder); - - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pTask); - return TSDB_CODE_INVALID_MSG; - } - - SStreamMeta* pStreamMeta = pTq->pStreamMeta; - - // 2.save task, use the latest commit version as the initial start version of stream task. - int32_t taskId = pTask->id.taskId; - int64_t streamId = pTask->id.streamId; - bool added = false; - - streamMetaWLock(pStreamMeta); - code = streamMetaRegisterTask(pStreamMeta, sversion, pTask, &added); - int32_t numOfTasks = streamMetaGetNumOfTasks(pStreamMeta); - streamMetaWUnLock(pStreamMeta); - - if (code < 0) { - tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); - tFreeStreamTask(pTask); - return code; - } - - // added into meta store, pTask cannot be reference since it may have been destroyed by other threads already now if - // it is added into the meta store - if (added) { - // only handled in the leader node - if (vnodeIsRoleLeader(pTq->pVnode)) { - tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); - SStreamTask* p = streamMetaAcquireTask(pStreamMeta, streamId, taskId); - - bool restored = pTq->pVnode->restored; - if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - streamTaskHandleEvent(p->status.pSM, event); - } else if (!restored) { - tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); - } - - if (p != NULL) { - streamMetaReleaseTask(pStreamMeta, p); - } - } else { - tqDebug("vgId:%d not leader, not launch stream task s-task:0x%x", vgId, taskId); - } - } else { - tqWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store", vgId, taskId); - tFreeStreamTask(pTask); - } - - return code; + return streamTaskProcessDeployReq(pTq->pStreamMeta, sversion, msg, msgLen, vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored); } static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { @@ -1244,190 +1086,39 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // only the agg tasks and the sink tasks will receive this message from upstream tasks int32_t tqProcessTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg) { - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - // deserialize - SStreamScanHistoryFinishReq req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeStreamScanHistoryFinishReq(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.downstreamTaskId); - if (pTask == NULL) { - tqError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", - pTq->pStreamMeta->vgId, req.downstreamTaskId); - return -1; - } - - tqDebug("s-task:%s receive scan-history finish msg from task:0x%x", pTask->id.idStr, req.upstreamTaskId); - - int32_t code = streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info); - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return code; + return streamTaskProcessScanHistoryFinishReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg) { - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - // deserialize - SStreamCompleteHistoryMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeCompleteHistoryDataMsg(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.upstreamTaskId); - if (pTask == NULL) { - tqError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", - pTq->pStreamMeta->vgId, req.upstreamTaskId); - return -1; - } - - int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); - if (remain > 0) { - tqDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", - pTask->id.idStr, req.downstreamId, remain); - } else { - tqDebug( - "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " - "completed msg", - pTask->id.idStr, req.downstreamId); - streamProcessScanHistoryFinishRsp(pTask); - } - - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; + return streamTaskProcessScanHistoryFinishRsp(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - int32_t vgId = TD_VID(pTq->pVnode); if (taskId == STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID) { // all tasks are extracted submit data from the wal tqScanWal(pTq); return 0; - } else if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { - tqStartStreamTasks(pTq); - return 0; - } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { - tqRestartStreamTasks(pTq); - return 0; } - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId); - if (pTask != NULL) { // even in halt status, the data in inputQ must be processed - char* p = NULL; - if (streamTaskReadyToRun(pTask, &p)) { - tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, - pTask->chkInfo.nextProcessVer); - streamExecTask(pTask); - } else { - int8_t status = streamTaskSetSchedStatusInactive(pTask); - tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, - pTask->id.idStr, p, status); - } - - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + int32_t code = streamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); + if(code == 0 && taskId > 0){ tqScanWalAsync(pTq, false); - return 0; - } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. - // todo add one function to handle this - tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, taskId); - return -1; } + return code; } -int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - SStreamDispatchReq req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamDispatchReq(&decoder, &req); - tDecoderClear(&decoder); - - tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.taskId); - if (pTask) { - SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessDispatchMsg(pTask, &req, &rsp); - tDeleteStreamDispatchReq(&req); - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; - } else { - tqError("vgId:%d failed to find task:0x%x to handle the dispatch req, it may have been destroyed already", - pTq->pStreamMeta->vgId, req.taskId); - tDeleteStreamDispatchReq(&req); - return -1; - } +int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { + return streamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { - SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - - int32_t vgId = pTq->pStreamMeta->vgId; - pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); - pRsp->streamId = htobe64(pRsp->streamId); - pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); - pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); - pRsp->stage = htobe64(pRsp->stage); - pRsp->msgId = htonl(pRsp->msgId); - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pRsp->streamId, pRsp->upstreamTaskId); - if (pTask) { - streamProcessDispatchRsp(pTask, pRsp, pMsg->code); - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return TSDB_CODE_SUCCESS; - } else { - tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); - terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; - return terrno; - } + return streamTaskProcessDispatchRsp(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { - SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - // drop the related fill-history task firstly - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); - tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId); - } - streamMetaReleaseTask(pMeta, pTask); - } - - // drop the stream task now - streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); - - // commit the update - streamMetaWLock(pMeta); - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - tqDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks); - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } - streamMetaWUnLock(pMeta); - - return 0; + return streamTaskProcessDropReq(pTq->pStreamMeta, msg, msgLen); } int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { @@ -1537,30 +1228,7 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms } int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - SDecoder decoder; - - SStreamRetrieveReq req; - tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamRetrieveReq(&decoder, &req); - tDecoderClear(&decoder); - - int32_t vgId = pTq->pStreamMeta->vgId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.dstTaskId); - if (pTask == NULL) { - tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", vgId, - req.dstTaskId); - return -1; - } - - SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessRetrieveReq(pTask, &req, &rsp); - - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - tDeleteStreamRetrieveReq(&req); - return 0; + return streamTaskProcessRetrieveReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) { @@ -1779,219 +1447,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) { - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; - - SStreamCheckpointReadyMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { - code = TSDB_CODE_MSG_DECODE_ERROR; - tDecoderClear(&decoder); - return code; - } - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); - if (pTask == NULL) { - tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); - return code; - } - - tqDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId, - pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); - - streamProcessCheckpointReadyMsg(pTask); - streamMetaReleaseTask(pMeta, pTask); - return code; + return streamTaskProcessCheckpointReadyMsg(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { - SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t vgId = TD_VID(pTq->pVnode); - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; - - SStreamTaskNodeUpdateMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { - rsp.code = TSDB_CODE_MSG_DECODE_ERROR; - tqError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); - tDecoderClear(&decoder); - return rsp.code; - } - - tDecoderClear(&decoder); - - // update the nodeEpset when it exists - streamMetaWLock(pMeta); - - // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. - STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - if (ppTask == NULL || *ppTask == NULL) { - tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, - req.taskId); - rsp.code = TSDB_CODE_SUCCESS; - streamMetaWUnLock(pMeta); - - taosArrayDestroy(req.pNodeList); - return rsp.code; - } - - SStreamTask* pTask = *ppTask; - - if (pMeta->updateInfo.transId != req.transId) { - pMeta->updateInfo.transId = req.transId; - tqInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); - // info needs to be kept till the new trans to update the nodeEp arrived. - taosHashClear(pMeta->updateInfo.pTasks); - } else { - tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); - } - - STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; - void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); - if (exist != NULL) { - tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, - req.transId); - rsp.code = TSDB_CODE_SUCCESS; - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return rsp.code; - } - - streamMetaWUnLock(pMeta); - - // the following two functions should not be executed within the scope of meta lock to avoid deadlock - streamTaskUpdateEpsetInfo(pTask, req.pNodeList); - streamTaskResetStatus(pTask); - - // continue after lock the meta again - streamMetaWLock(pMeta); - - SStreamTask** ppHTask = NULL; - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); - if (ppHTask == NULL || *ppHTask == NULL) { - tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - pMeta->vgId, req.taskId); - CLEAR_RELATED_FILLHISTORY_TASK(pTask); - } else { - tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); - streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); - } - } - - { - streamMetaSaveTask(pMeta, pTask); - if (ppHTask != NULL) { - streamMetaSaveTask(pMeta, *ppHTask); - } - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } - } - - streamTaskStop(pTask); - - // keep the already handled info - taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); - - if (ppHTask != NULL) { - streamTaskStop(*ppHTask); - tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); - taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); - } else { - tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); - } - - rsp.code = 0; - - // possibly only handle the stream task. - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); - - pMeta->startInfo.tasksWillRestart = 1; - - if (updateTasks < numOfTasks) { - tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, - updateTasks, (numOfTasks - updateTasks)); - streamMetaWUnLock(pMeta); - } else { - if (!pTq->pVnode->restored) { - tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); - pMeta->startInfo.tasksWillRestart = 0; - streamMetaWUnLock(pMeta); - } else { - tqDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); - -#if 1 - tqStartStreamTaskAsync(pTq, true); - streamMetaWUnLock(pMeta); -#else - streamMetaWUnLock(pMeta); - - // For debug purpose. - // the following procedure consume many CPU resource, result in the re-election of leader - // with high probability. So we employ it as a test case for the stream processing framework, with - // checkpoint/restart/nodeUpdate etc. - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - int32_t code = streamMetaReopen(pMeta); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - streamMetaInitBackend(pMeta); - - if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { - tqError("vgId:%d failed to load stream tasks", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - tqResetStreamTaskStatus(pTq); - tqStartStreamTaskAsync(pTq, false); - } else { - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - streamMetaWUnLock(pMeta); -#endif - } - } - - taosArrayDestroy(req.pNodeList); - return rsp.code; + return streamTaskProcesUpdateReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, pMsg, pTq->pVnode->restored); } int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index e069d6c4b7..4776447e31 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -15,7 +15,6 @@ #include "tq.h" #include "vnd.h" -#include "stream.h" #define MAX_REPEAT_SCAN_THRESHOLD 3 #define SCAN_WAL_IDLE_DURATION 100 @@ -61,154 +60,6 @@ int32_t tqScanWal(STQ* pTq) { return 0; } -int32_t tqStartStreamTasks(STQ* pTq) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - taosHashClear(pMeta->startInfo.pReadyTaskSet); - taosHashClear(pMeta->startInfo.pFailedTaskSet); - pMeta->startInfo.startTs = taosGetTimestampMs(); - streamMetaWUnLock(pMeta); - - // broadcast the check downstream tasks msg - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - continue; - } - - // fill-history task can only be launched by related stream tasks. - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - if (pTask->status.downstreamReady == 1) { - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); - } - - streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); - if (ret != TSDB_CODE_SUCCESS) { - code = ret; - } - - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - return code; -} - -int32_t tqRestartStreamTasks(STQ* pTq) { - SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t vgId = pMeta->vgId; - int32_t code = 0; - int64_t st = taosGetTimestampMs(); - - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - terrno = 0; - tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, - pMeta->updateInfo.transId); - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - code = streamMetaReopen(pMeta); - if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - code = terrno; - return code; - } - - streamMetaInitBackend(pMeta); - int64_t el = taosGetTimestampMs() - st; - - tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); - - code = streamMetaLoadAllTasks(pTq->pStreamMeta); - if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); - streamMetaWUnLock(pMeta); - code = terrno; - return code; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); - tqResetStreamTaskStatus(pTq); - - streamMetaWUnLock(pMeta); - tqStartStreamTasks(pTq); - } else { - streamMetaResetStartInfo(&pMeta->startInfo); - streamMetaWUnLock(pMeta); - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - - code = terrno; - return code; -} - -int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart) { - SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t vgId = pMeta->vgId; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - tqDebug("vgId:%d no stream tasks existed to run", vgId); - return 0; - } - - SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - return -1; - } - - tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); - pRunReq->head.vgId = vgId; - pRunReq->streamId = 0; - pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); - return 0; -} - int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; @@ -298,27 +149,6 @@ int32_t tqStopStreamTasks(STQ* pTq) { return 0; } -int32_t tqResetStreamTaskStatus(STQ* pTq) { - SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t vgId = TD_VID(pTq->pVnode); - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - - tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); - - STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; - SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - streamTaskResetStatus(*pTask); - } - - return 0; -} - int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId) { // seek the stored version and extract data from WAL int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 33b4114009..9b79e9bd66 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -463,7 +463,6 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { break; } -_exit: if (code) { vError("vgId:%d, failed to preprocess write request since %s, msg type:%s", TD_VID(pVnode), tstrerror(code), TMSG_INFO(pMsg->msgType)); @@ -762,7 +761,7 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: - return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); + return tqProcessTaskDispatchReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); case TDMT_VND_STREAM_TASK_CHECK: diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 3944f8ed91..1f0f8f7362 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -569,8 +569,8 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); - tqResetStreamTaskStatus(pVnode->pTq); - tqStartStreamTaskAsync(pVnode->pTq, false); + resetStreamTaskStatus(pVnode->pTq->pStreamMeta); + streamTaskStartAsync(pMeta, &pVnode->msgCb, false); } } else { vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index b76a967d0d..b9a106321e 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -52,6 +52,12 @@ extern "C" { #define stTrace(...) do { if (stDebugFlag & DEBUG_TRACE) { taosPrintLog("STM ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +typedef struct STaskUpdateEntry { + int64_t streamId; + int32_t taskId; + int32_t transId; +} STaskUpdateEntry; + typedef struct { int8_t type; SSDataBlock* pBlock; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index ab7951bb92..1c874f34de 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -276,6 +276,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code)); + terrno = code; return code; } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index a1c0fa2040..3474833502 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -411,7 +411,6 @@ static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { ASSERT(pTask->id.taskId == pRsp->upstreamTaskId); const char* id = pTask->id.idStr; - int32_t vgId = pTask->pMeta->vgId; if (streamTaskShouldStop(pTask)) { stDebug("s-task:%s should stop, do not do check downstream again", id); @@ -463,14 +462,15 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%"PRId64", current stage:%"PRId64", " "not check wait for downstream task nodeUpdate, and all tasks restart", id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); + addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " "downstream again, nodeUpdate needed", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); + addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } - addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, taosGetTimestampMs(), false); } else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 24228c0307..143792d949 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -779,4 +779,787 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->sinkDataSize = pSrc->sinkDataSize; pDst->activeCheckpointId = pSrc->activeCheckpointId; pDst->checkpointFailed = pSrc->checkpointFailed; -} \ No newline at end of file +} + +int32_t streamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + stDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + stError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + return -1; + } + + stDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(cb, STREAM_QUEUE, &msg); + return 0; +} + +int32_t streamTaskProcesUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { + int32_t vgId = pMeta->vgId; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; + + SStreamTaskNodeUpdateMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { + rsp.code = TSDB_CODE_MSG_DECODE_ERROR; + stError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); + tDecoderClear(&decoder); + return rsp.code; + } + + tDecoderClear(&decoder); + + // update the nodeEpset when it exists + streamMetaWLock(pMeta); + + // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. + STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask == NULL || *ppTask == NULL) { + stError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, + req.taskId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + SStreamTask* pTask = *ppTask; + + if (pMeta->updateInfo.transId != req.transId) { + pMeta->updateInfo.transId = req.transId; + stInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); + // info needs to be kept till the new trans to update the nodeEp arrived. + taosHashClear(pMeta->updateInfo.pTasks); + } else { + stDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); + } + + STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; + void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); + if (exist != NULL) { + stDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, + req.transId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + streamMetaWUnLock(pMeta); + + // the following two functions should not be executed within the scope of meta lock to avoid deadlock + streamTaskUpdateEpsetInfo(pTask, req.pNodeList); + streamTaskResetStatus(pTask); + + // continue after lock the meta again + streamMetaWLock(pMeta); + + SStreamTask** ppHTask = NULL; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); + if (ppHTask == NULL || *ppHTask == NULL) { + stError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", + pMeta->vgId, req.taskId); + CLEAR_RELATED_FILLHISTORY_TASK(pTask); + } else { + stDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); + streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); + } + } + + { + streamMetaSaveTask(pMeta, pTask); + if (ppHTask != NULL) { + streamMetaSaveTask(pMeta, *ppHTask); + } + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + } + + streamTaskStop(pTask); + + // keep the already handled info + taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); + + if (ppHTask != NULL) { + streamTaskStop(*ppHTask); + stDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); + taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); + } else { + stDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); + } + + rsp.code = 0; + + // possibly only handle the stream task. + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); + + pMeta->startInfo.tasksWillRestart = 1; + + if (updateTasks < numOfTasks) { + stDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, + updateTasks, (numOfTasks - updateTasks)); + streamMetaWUnLock(pMeta); + } else { + if (!restored) { + stDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); + pMeta->startInfo.tasksWillRestart = 0; + streamMetaWUnLock(pMeta); + } else { + stDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); +#if 1 + streamTaskStartAsync(pMeta, cb, true); + streamMetaWUnLock(pMeta); +#else + streamMetaWUnLock(pMeta); + + // For debug purpose. + // the following procedure consume many CPU resource, result in the re-election of leader + // with high probability. So we employ it as a test case for the stream processing framework, with + // checkpoint/restart/nodeUpdate etc. + while (1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + while (streamMetaTaskInTimer(pMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + int32_t code = streamMetaReopen(pMeta); + if (code != 0) { + tqError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + streamMetaInitBackend(pMeta); + + if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { + tqError("vgId:%d failed to load stream tasks", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { + tqInfo("vgId:%d start all stream tasks after all being updated", vgId); + resetStreamTaskStatus(pTq->pStreamMeta); + tqStartStreamTaskAsync(pTq, false); + } else { + tqInfo("vgId:%d, follower node not start stream tasks", vgId); + } + streamMetaWUnLock(pMeta); +#endif + } + } + + taosArrayDestroy(req.pNodeList); + return rsp.code; +} + +int32_t streamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamDispatchReq req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); + if (tDecodeStreamDispatchReq(&decoder, &req) < 0) { + tDecoderClear(&decoder); + return TSDB_CODE_MSG_DECODE_ERROR; + } + tDecoderClear(&decoder); + + stDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); + if (pTask) { + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; + if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0){ + return -1; + } + tDeleteStreamDispatchReq(&req); + streamMetaReleaseTask(pMeta, pTask); + return 0; + } else { + stError("vgId:%d failed to find task:0x%x to handle the dispatch req, it may have been destroyed already", + pMeta->vgId, req.taskId); + + SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); + if (pRspHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + stError("s-task:0x%x send dispatch error rsp, out of memory", req.taskId); + return -1; + } + + pRspHead->vgId = htonl(req.upstreamNodeId); + ASSERT(pRspHead->vgId != 0); + + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead)); + pRsp->streamId = htobe64(req.streamId); + pRsp->upstreamTaskId = htonl(req.upstreamTaskId); + pRsp->upstreamNodeId = htonl(req.upstreamNodeId); + pRsp->downstreamNodeId = htonl(pMeta->vgId); + pRsp->downstreamTaskId = htonl(req.taskId); + pRsp->msgId = htonl(req.msgId); + pRsp->stage = htobe64(req.stage); + pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; + + int32_t len = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); + SRpcMsg rsp = {.code = TSDB_CODE_STREAM_TASK_NOT_EXIST, .info = pMsg->info, .contLen = len, .pCont = pRspHead}; + stError("s-task:0x%x send dispatch error rsp, no task", req.taskId); + + tmsgSendRsp(&rsp); + tDeleteStreamDispatchReq(&req); + + return 0; + } +} + +int32_t streamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + + int32_t vgId = pMeta->vgId; + pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); + pRsp->streamId = htobe64(pRsp->streamId); + pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); + pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); + pRsp->stage = htobe64(pRsp->stage); + pRsp->msgId = htonl(pRsp->msgId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); + if (pTask) { + streamProcessDispatchRsp(pTask, pRsp, pMsg->code); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; + } else { + stDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); + terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; + return terrno; + } +} + +int32_t streamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder; + + SStreamRetrieveReq req; + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); + tDecodeStreamRetrieveReq(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId); + if (pTask == NULL) { + stError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, + req.dstTaskId); + return -1; + } + + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; + streamProcessRetrieveReq(pTask, &req, &rsp); + + streamMetaReleaseTask(pMeta, pTask); + tDeleteStreamRetrieveReq(&req); + return 0; +} + +int32_t streamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamScanHistoryFinishReq req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.downstreamTaskId); + if (pTask == NULL) { + stError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", + pMeta->vgId, req.downstreamTaskId); + return -1; + } + + stDebug("s-task:%s receive scan-history finish msg from task:0x%x", pTask->id.idStr, req.upstreamTaskId); + + int32_t code = streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + +int32_t streamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamCompleteHistoryMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeCompleteHistoryDataMsg(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + stError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", + pMeta->vgId, req.upstreamTaskId); + return -1; + } + + int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); + if (remain > 0) { + stDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", + pTask->id.idStr, req.downstreamId, remain); + } else { + stDebug( + "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " + "completed msg", + pTask->id.idStr, req.downstreamId); + streamProcessScanHistoryFinishRsp(pTask); + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + +int32_t streamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamTaskCheckReq req; + SDecoder decoder; + + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); + tDecodeStreamTaskCheckReq(&decoder, &req); + tDecoderClear(&decoder); + + int32_t taskId = req.downstreamTaskId; + + SStreamTaskCheckRsp rsp = { + .reqId = req.reqId, + .streamId = req.streamId, + .childId = req.childId, + .downstreamNodeId = req.downstreamNodeId, + .downstreamTaskId = req.downstreamTaskId, + .upstreamNodeId = req.upstreamNodeId, + .upstreamTaskId = req.upstreamTaskId, + }; + + // only the leader node handle the check request + if (pMeta->role == NODE_ROLE_FOLLOWER) { + stError("s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", + taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); + rsp.status = TASK_DOWNSTREAM_NOT_LEADER; + } else { + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); + if (pTask != NULL) { + rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); + streamMetaReleaseTask(pMeta, pTask); + + char* p = NULL; + streamTaskGetStatus(pTask, &p); + stDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", + pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } else { + rsp.status = TASK_DOWNSTREAM_NOT_READY; + stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 + ") from task:0x%x (vgId:%d), rsp check_status %d", + req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } + } + + return streamSendCheckRsp(pMeta, &req, &rsp, &pMsg->info, taskId); +} + +int32_t streamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t vgId = pMeta->vgId; + + int32_t code; + SStreamTaskCheckRsp rsp; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pReq, len); + code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); + if (code < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&decoder); + stError("vgId:%d failed to parse check rsp msg, code:%s", vgId, tstrerror(terrno)); + return -1; + } + + tDecoderClear(&decoder); + stDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + + if (!isLeader) { + stError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, + rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); + return code; + } + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); + if (pTask == NULL) { + stError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", + rsp.streamId, rsp.upstreamTaskId, vgId); + terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; + return -1; + } + + code = streamProcessCheckRsp(pTask, &rsp); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + +int32_t streamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { + int32_t vgId = pMeta->vgId; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; + + SStreamCheckpointReadyMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + tDecoderClear(&decoder); + return code; + } + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + stError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); + return code; + } + + stDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId, + pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); + + streamProcessCheckpointReadyMsg(pTask); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + +int32_t streamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored) { + int32_t code = 0; + int32_t vgId = pMeta->vgId; + + if (tsDisableStream) { + stInfo("vgId:%d stream disabled, not deploy stream tasks", vgId); + return code; + } + + stDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId); + + // 1.deserialize msg and build task + int32_t size = sizeof(SStreamTask); + SStreamTask* pTask = taosMemoryCalloc(1, size); + if (pTask == NULL) { + stError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + code = tDecodeStreamTask(&decoder, pTask); + tDecoderClear(&decoder); + + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pTask); + return TSDB_CODE_INVALID_MSG; + } + + // 2.save task, use the latest commit version as the initial start version of stream task. + int32_t taskId = pTask->id.taskId; + int64_t streamId = pTask->id.streamId; + bool added = false; + + streamMetaWLock(pMeta); + code = streamMetaRegisterTask(pMeta, sversion, pTask, &added); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + streamMetaWUnLock(pMeta); + + if (code < 0) { + stError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); + tFreeStreamTask(pTask); + return code; + } + + // added into meta store, pTask cannot be reference since it may have been destroyed by other threads already now if + // it is added into the meta store + if (added) { + // only handled in the leader node + if (isLeader) { + stDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); + SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); + + if (p != NULL && restored && p->info.fillHistory == 0) { + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + streamTaskHandleEvent(p->status.pSM, event); + } else if (!restored) { + stWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); + } + + if (p != NULL) { + streamMetaReleaseTask(pMeta, p); + } + } else { + stDebug("vgId:%d not leader, not launch stream task s-task:0x%x", vgId, taskId); + } + } else { + stWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store", vgId, taskId); + tFreeStreamTask(pTask); + } + + return code; +} + +int32_t streamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { + SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + + int32_t vgId = pMeta->vgId; + stDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask != NULL) { + // drop the related fill-history task firstly + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pHTaskId = &pTask->hTaskInfo.id; + streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); + stDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId); + } + streamMetaReleaseTask(pMeta, pTask); + } + + // drop the stream task now + streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); + + // commit the update + streamMetaWLock(pMeta); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + stDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks); + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + streamMetaWUnLock(pMeta); + + return 0; +} + +int32_t startStreamTasks(SStreamMeta* pMeta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + stDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.startTs = taosGetTimestampMs(); + streamMetaWUnLock(pMeta); + + // broadcast the check downstream tasks msg + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + // fill-history task can only be launched by related stream tasks. + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + if (pTask->status.downstreamReady == 1) { + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", + pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return code; +} + +int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + + stDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + + STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; + SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + streamTaskResetStatus(*pTask); + } + + return 0; +} + +static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { + int32_t vgId = pMeta->vgId; + int32_t code = 0; + int64_t st = taosGetTimestampMs(); + + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + stDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + terrno = 0; + stInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, + pMeta->updateInfo.transId); + + while (streamMetaTaskInTimer(pMeta)) { + stDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + code = streamMetaReopen(pMeta); + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + streamMetaInitBackend(pMeta); + int64_t el = taosGetTimestampMs() - st; + + stInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); + + code = streamMetaLoadAllTasks(pMeta); + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + if (isLeader && !tsDisableStream) { + stInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + resetStreamTaskStatus(pMeta); + + streamMetaWUnLock(pMeta); + startStreamTasks(pMeta); + } else { + streamMetaResetStartInfo(&pMeta->startInfo); + streamMetaWUnLock(pMeta); + stInfo("vgId:%d, follower node not start stream tasks", vgId); + } + + code = terrno; + return code; +} + +int32_t streamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { + SStreamTaskRunReq* pReq = pMsg->pCont; + + int32_t taskId = pReq->taskId; + int32_t vgId = pMeta->vgId; + + if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { + startStreamTasks(pMeta); + return 0; + } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { + restartStreamTasks(pMeta, isLeader); + return 0; + } + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); + if (pTask != NULL) { // even in halt status, the data in inputQ must be processed + char* p = NULL; + if (streamTaskReadyToRun(pTask, &p)) { + stDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, + pTask->chkInfo.nextProcessVer); + streamExecTask(pTask); + } else { + int8_t status = streamTaskSetSchedStatusInactive(pTask); + stDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, + pTask->id.idStr, p, status); + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; + } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. + // todo add one function to handle this + stError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, taskId); + return -1; + } +} From 6dc91d12d557f7f11177350f2dae5f077da99d69 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 22 Nov 2023 20:23:58 +0800 Subject: [PATCH 10/43] fix:add msg type --- include/common/tmsgdef.h | 22 +++++++++++----------- source/dnode/snode/src/snode.c | 3 +++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 4eb8328caa..ba0217813c 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -67,7 +67,7 @@ enum { enum { // WARN: new msg should be appended to segment tail #endif - TD_NEW_MSG_SEG(TDMT_DND_MSG) + TD_NEW_MSG_SEG(TDMT_DND_MSG) // 0<<8 TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL) @@ -87,7 +87,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_MND_MSG) + TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8 TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ACCT, "create-acct", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_ACCT, "alter-acct", NULL, NULL) @@ -194,7 +194,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_VND_MSG) + TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TABLE, "create-table", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "alter-table", NULL, NULL) @@ -243,7 +243,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_SCH_MSG) + TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8 TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MERGE_QUERY, "merge-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_CONTINUE, "query-continue", NULL, NULL) @@ -258,13 +258,13 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_STREAM_MSG) - TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) + TD_NEW_MSG_SEG(TDMT_STREAM_MSG) //4 << 8 + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) //1025 1026 TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) //1035 1036 TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT_READY, "stream-checkpoint-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL) @@ -273,10 +273,10 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_MON_MSG) + TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_SYNC_MSG) + TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8 TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used @@ -307,7 +307,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) + TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8 TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY_FINISH, "vnode-stream-scan-history-finish", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL) @@ -316,7 +316,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) + TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8 TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 1c8e3e9bef..65c1e3d450 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -172,6 +172,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return streamTaskProcessDispatchRsp(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE: return streamTaskProcessRetrieveReq(pSnode->pMeta, pMsg); + case TDMT_STREAM_RETRIEVE_RSP: // 1036 + break; case TDMT_VND_STREAM_SCAN_HISTORY_FINISH: return streamTaskProcessScanHistoryFinishReq(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP: @@ -183,6 +185,7 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { case TDMT_STREAM_TASK_CHECKPOINT_READY: return streamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg); default: + sndError("invalid snode msg:%d", pMsg->msgType); ASSERT(0); } return 0; From 15d75a0497915487580ef3c71101d0d531269d4a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 23 Nov 2023 09:17:54 +0800 Subject: [PATCH 11/43] fix:spelling errors --- include/libs/stream/tstream.h | 2 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/stream/src/streamTask.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 69a44473f0..0014335741 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -855,7 +855,7 @@ int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInf // message process int32_t streamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart); -int32_t streamTaskProcesUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored); +int32_t streamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored); int32_t streamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t streamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t streamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 65c1e3d450..8db2e783db 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -202,7 +202,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DROP: return streamTaskProcessDropReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); case TDMT_VND_STREAM_TASK_UPDATE: - return streamTaskProcesUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); + return streamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5daef7d5db..90badaa5c6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1451,7 +1451,7 @@ int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcesUpdateReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, pMsg, pTq->pVnode->restored); + return streamTaskProcessUpdateReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, pMsg, pTq->pVnode->restored); } int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 143792d949..6ff0581a31 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -807,7 +807,7 @@ int32_t streamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { return 0; } -int32_t streamTaskProcesUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { +int32_t streamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t vgId = pMeta->vgId; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); From 55a4181e5e5a6253803af754447d2c71af433c6b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 23 Nov 2023 20:16:54 +0800 Subject: [PATCH 12/43] fix:add O_CLOEXEC for .running lock file to prevent inherited by child processes(rsync) --- include/os/osFile.h | 2 ++ source/common/src/rsync.c | 2 ++ source/dnode/mgmt/node_util/src/dmFile.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- source/os/src/osFile.c | 2 ++ 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index e409936468..503535a454 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -64,6 +64,8 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_EXCL 0x0080 #define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile #define TD_FILE_WRITE_THROUGH 0x0200 +#define TD_FILE_CLOEXEC 0x0400 + TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions); TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions); diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index ffab85761e..6f6a021b3b 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -151,6 +151,8 @@ void startRsync(){ uDebug("[rsync] start server successful"); } +} + int uploadRsync(char* id, char* path){ #ifdef WINDOWS char pathTransform[PATH_MAX] = {0}; diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 03c6734e0c..5cbeeebc14 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -149,7 +149,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP); - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to open file:%s since %s", filepath, terrstr()); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28832ffec8..5d4d2b7461 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1549,7 +1549,7 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; - char* tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0}; + char tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0}; if (groupId == 0) { groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index cf3bce1ad4..15aca85fc2 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -647,6 +647,8 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + access |= (tdFileOptions & TD_FILE_CLOEXEC) ? O_CLOEXEC : 0; + int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); return fd; } From f28cf740b09fc4e0d539f3b722f7c5216e56a174 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 24 Nov 2023 14:44:00 +0800 Subject: [PATCH 13/43] fix(stream): fix status check for stream-scan-history status when drop stream task msg is missing. --- include/libs/stream/tstream.h | 2 ++ source/dnode/mnode/impl/src/mndStream.c | 19 +++++++++++++++-- source/util/src/tdecompress.c | 27 +++++++++++++------------ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index cf9fc1d826..b8ceb31b80 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -664,6 +664,8 @@ typedef struct STaskStatusEntry { int32_t relatedHTask; // has related fill-history task int64_t activeCheckpointId; // current active checkpoint id bool checkpointFailed; // denote if the checkpoint is failed or not + bool inputQChanging; // inputQ is changing or not + int64_t inputQUnchangeCounter; double inputQUsed; // in MiB double inputRate; double sinkQuota; // existed quota size for sink task diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index db013017e3..5490d53587 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2790,7 +2790,7 @@ static SStreamTask* mndGetStreamTask(STaskId* pId, SStreamObj* pStream) { static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) { if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) { - if (fabs(pTaskEntry->inputQUsed) <= DBL_EPSILON) { + if (!pTaskEntry->inputQChanging && pTaskEntry->inputQUnchangeCounter > 10) { int32_t numOfReady = 0; int32_t numOfTotal = 0; for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { @@ -2933,6 +2933,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { bool snodeChanged = false; for (int32_t i = 0; i < req.numOfTasks; ++i) { STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i); + STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id)); if (pTaskEntry == NULL) { mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId); @@ -2941,8 +2942,22 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) { updateStageInfo(pTaskEntry, p->stage); - if(pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; + if(pTaskEntry->nodeId == SNODE_HANDLE) { + snodeChanged = true; + } } else { + // task is idle for more than 50 sec. + if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { + if (!pTaskEntry->inputQChanging) { + pTaskEntry->inputQUnchangeCounter++; + } else { + pTaskEntry->inputQChanging = false; + } + } else { + pTaskEntry->inputQChanging = true; + pTaskEntry->inputQUnchangeCounter = 0; + } + streamTaskStatusCopy(pTaskEntry, p); if (p->activeCheckpointId != 0) { if (activeCheckpointId != 0) { diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 68841941db..19f4e1fd92 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -53,11 +53,8 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, int64_t prevValue = 0; #if __AVX2__ - while (1) { - if (_pos == nelements) break; - - uint64_t w = 0; - memcpy(&w, ip, LONG_BYTES); + while (_pos < nelements) { + uint64_t w = *(uint64_t*) ip; char selector = (char)(w & INT64MASK(4)); // selector = 4 char bit = bit_per_integer[(int32_t)selector]; // bit = 3 @@ -261,17 +258,20 @@ int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelemen int64_t deltaOfDelta = 0; int32_t longBytes = LONG_BYTES; + // _mm_maskz_loadu_epi8 #if __AVX2__ - int32_t batch = nelements >> 2; - int32_t remainder = nelements & 0x1; + // _mm_blendv_epi8 + int32_t batch = nelements >> 4; + int32_t remainder = nelements & 0x03; - while (1) { + for(int32_t i = 0; i < batch; ++i) { uint8_t flags = input[ipos++]; // Decode dd1 uint64_t dd1 = 0; - nbytes = flags & INT8MASK(4); + nbytes = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + // __m128i mask = {};//[0], [] if (nbytes == 0) { deltaOfDelta = 0; @@ -289,10 +289,6 @@ int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelemen prevValue += prevDelta; ostream[opos++] = prevValue; - if (opos == nelements) { - return nelements * longBytes; - } - // Decode dd2 uint64_t dd2 = 0; nbytes = (flags >> 4) & INT8MASK(4); @@ -317,6 +313,11 @@ int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelemen return nelements * longBytes; } } + + if (remainder > 0) { + + } + #endif return 0; } \ No newline at end of file From 3deab3cfbfdfde63bf550f46bbf8d948e6c4c10f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 24 Nov 2023 15:37:30 +0800 Subject: [PATCH 14/43] fix(stream):set failure for the related fill-history task. --- source/libs/stream/src/streamStart.c | 13 ++++++++++++- source/util/src/tdecompress.c | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 469813defc..ce18e770b1 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -471,6 +471,14 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, taosGetTimestampMs(), false); + // automatically set the related fill-history task to be failed. + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + + SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pId->streamId, pId->taskId); + streamMetaUpdateTaskDownstreamStatus(pHTask, pHTask->execInfo.init, taosGetTimestampMs(), false); + streamMetaReleaseTask(pTask->pMeta, pHTask); + } } else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp); @@ -1072,8 +1080,9 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta); + int32_t numOfRecv = taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet); - if (taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet) == numOfTotal) { + if (numOfRecv == numOfTotal) { pStartInfo->readyTs = taosGetTimestampMs(); pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; @@ -1087,6 +1096,8 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); streamMetaResetStartInfo(pStartInfo); + } else { + stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } streamMetaWUnLock(pMeta); diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 19f4e1fd92..ac9d70b2e7 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -259,7 +259,7 @@ int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelemen int32_t longBytes = LONG_BYTES; // _mm_maskz_loadu_epi8 -#if __AVX2__ +#if __AVX512F__ // _mm_blendv_epi8 int32_t batch = nelements >> 4; From f8477204e1e79a6b92117aa5b651f6b346b2a2e8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 24 Nov 2023 18:00:19 +0800 Subject: [PATCH 15/43] fix:[TD-27475] send msg to snode wher drop stream to avoid pVgObj is NULL --- source/dnode/mnode/impl/src/mndStream.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d2f0a13038..c14d99ca36 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -18,6 +18,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndSnode.h" #include "mndPrivilege.h" #include "mndScheduler.h" #include "mndShow.h" @@ -695,9 +696,15 @@ static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask pReq->streamId = pTask->id.streamId; STransAction action = {0}; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset = {0}; + if(pTask->info.nodeId == SNODE_HANDLE){ + SSnodeObj* pObj = mndAcquireSnode(pMnode, pTask->info.nodeId); + addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + }else{ + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0); From 788194c74890585fbbf3dcdbc83a93337e6b3db0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 24 Nov 2023 22:53:55 +0800 Subject: [PATCH 16/43] fix(stream): fix deadlock. --- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 8 ++-- source/dnode/mnode/impl/src/mndStreamTrans.c | 41 ++++++++++++-------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 244a6d08dd..f2b3ce6cac 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -52,7 +52,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pName, const char* pSrcDb, const char* pDstDb); -bool streamTransConflictOtherTrans(SMnode *pMnode, const char *pSrcDb, const char *pDstDb); +bool streamTransConflictOtherTrans(SMnode *pMnode, const char *pSrcDb, const char *pDstDb, bool lock); // for sma // TODO refactor diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5490d53587..81cb4703e1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1348,7 +1348,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); tFreeMDropStreamReq(&dropReq); @@ -1831,7 +1831,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -1966,7 +1966,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -2754,7 +2754,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { break; } - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb, false); if (conflict) { mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name, pStream->sourceDb, pStream->targetDb); diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 2345de290a..9dd9f64037 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -35,17 +35,15 @@ int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pName, const char* pS } int32_t clearFinishedTrans(SMnode* pMnode) { - SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); size_t keyLen = 0; + SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); + void* pIter = NULL; - taosThreadMutexLock(&execInfo.lock); - - void* pIter = NULL; while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { - SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter; - STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); + SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter; // let's clear the finished trans + STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); if (pTrans == NULL) { void* pKey = taosHashGetKey(pEntry, &keyLen); // key is the name of src/dst db name @@ -60,44 +58,55 @@ int32_t clearFinishedTrans(SMnode* pMnode) { } size_t num = taosArrayGetSize(pList); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { SKeyInfo* pKey = taosArrayGet(pList, i); taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); } - mDebug("clear %d finished stream-trans, remained:%d", (int32_t) num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); - taosThreadMutexUnlock(&execInfo.lock); + mDebug("clear %d finished stream-trans, remained:%d", (int32_t)num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); terrno = TSDB_CODE_SUCCESS; taosArrayDestroy(pList); return 0; } -bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const char* pDstDb) { - clearFinishedTrans(pMnode); +bool streamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const char* pDstDb, bool lock) { + if (lock) { + taosThreadMutexLock(&execInfo.lock); + } - taosThreadMutexLock(&execInfo.lock); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } return false; } + clearFinishedTrans(pMnode); + SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb)); if (pEntry != NULL) { - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name); return true; } pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pDstDb, strlen(pDstDb)); if (pEntry != NULL) { - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name); return true; } - taosThreadMutexUnlock(&execInfo.lock); + if (lock) { + taosThreadMutexUnlock(&execInfo.lock); + } + return false; } From 30c776cc0a2c82a2aafeb3f865caf8ffd575d310 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 25 Nov 2023 00:54:32 +0800 Subject: [PATCH 17/43] fix(stream): reset the scheduler status. --- source/dnode/vnode/src/tq/tq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9d16402ee6..821adc9953 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -2041,6 +2041,8 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); + // clear the scheduler status + streamTaskSetSchedStatusInactive(pTask); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } From 9194c0c0cd929f1d78cddb7fbd532170d0756292 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 25 Nov 2023 00:56:18 +0800 Subject: [PATCH 18/43] fix(stream): add logs. --- source/dnode/vnode/src/tq/tq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 821adc9953..de8305e6d9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -2043,6 +2043,7 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { // clear the scheduler status streamTaskSetSchedStatusInactive(pTask); + tqDebug("s-task:%s set scheduler status:%d after drop fill-history task", pTask->id.idStr, pTask->status.schedStatus); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } From 1dd5cd17a04a4785026e48291c8088d1431850cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 26 Nov 2023 23:53:05 +0800 Subject: [PATCH 19/43] fix(tsdb): add simd for decompress timestamp. --- cmake/cmake.define | 4 +- include/util/tcompression.h | 1 + source/util/src/tdecompress.c | 148 ++++++++++++++++++++++------------ 3 files changed, 100 insertions(+), 53 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 7710c071eb..3eb872cfee 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -181,8 +181,8 @@ ELSE () MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") MESSAGE(STATUS "avx512 supported by gcc") ENDIF() ENDIF() diff --git a/include/util/tcompression.h b/include/util/tcompression.h index ab0c22fc9b..7da3587f1c 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -139,6 +139,7 @@ int32_t getWordLength(char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); +int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian); /************************************************************************* * STREAM COMPRESSION diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index ac9d70b2e7..b1c6df95c4 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -247,75 +247,121 @@ int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelemen return 0; } -int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelements, char *const output, bool bigEndian) { +int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; - int8_t nbytes = 0; - - int64_t prevValue = 0; - int64_t prevDelta = 0; - - int64_t deltaOfDelta = 0; - int32_t longBytes = LONG_BYTES; + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); // _mm_maskz_loadu_epi8 #if __AVX512F__ - // _mm_blendv_epi8 - int32_t batch = nelements >> 4; - int32_t remainder = nelements & 0x03; + int32_t batch = nelements >> 1; + int32_t remainder = nelements & 0x01; - for(int32_t i = 0; i < batch; ++i) { + int32_t i = 0; + if (batch > 1) { + // first loop uint8_t flags = input[ipos++]; - // Decode dd1 - uint64_t dd1 = 0; - nbytes = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 - // __m128i mask = {};//[0], [] + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - if (nbytes == 0) { - deltaOfDelta = 0; - } else { - if (bigEndian) { - memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd1, input + ipos, nbytes); - } - deltaOfDelta = ZIGZAG_DECODE(int64_t, dd1); - } + __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + data2 = _mm_broadcastq_epi64(data2); - ipos += nbytes; - prevDelta += deltaOfDelta; - prevValue += prevDelta; - ostream[opos++] = prevValue; + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); - // Decode dd2 - uint64_t dd2 = 0; - nbytes = (flags >> 4) & INT8MASK(4); - if (nbytes == 0) { - deltaOfDelta = 0; - } else { - if (bigEndian) { - memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd2, input + ipos, nbytes); - } - // zigzag_decoding - deltaOfDelta = ZIGZAG_DECODE(int64_t, dd2); - } + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); - ipos += nbytes; - prevDelta += deltaOfDelta; - prevValue += prevDelta; - ostream[opos++] = prevValue; + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); - if (opos == nelements) { - return nelements * longBytes; - } + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_set1_epi64x(val[1]); + + // keep the previous delta of delta, for the first item + prevDelta = _mm_set1_epi64x(deltaOfDelta[1]); + + opos += 2; + ipos += nbytes1 + nbytes2; + i += 1; + } + + // the remain + for(; i < batch; ++i) { + uint8_t flags = input[ipos++]; + + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + + __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + data2 = _mm_broadcastq_epi64(data2); + + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_set1_epi64x(val[1]); + + // keep the previous delta of delta + __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta); + prevDelta = _mm_set1_epi64x(_mm_add_epi64(delta, prevDelta)[1]); + + opos += 2; + ipos += nbytes1 + nbytes2; } if (remainder > 0) { + uint64_t dd = 0; + uint8_t flags = input[ipos++]; + int32_t nbytes = flags & INT8MASK(4); + int64_t deltaOfDelta = 0; + if (nbytes == 0) { + deltaOfDelta = 0; + } else { + // if (is_bigendian()) { + // memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + // } else { + memcpy(&dd, input + ipos, nbytes); + // } + deltaOfDelta = ZIGZAG_DECODE(int64_t, dd); + } + + ipos += nbytes; + if (opos == 0) { + ostream[opos++] = deltaOfDelta; + } else { + int64_t prevV = prevVal[1]; + + int64_t prevDeltaX = deltaOfDelta + prevDelta[1]; + ostream[opos++] = prevV + prevDeltaX; + } } #endif From de0c20c296ae78e4e16ee2a869e1524614e9c0a2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 00:02:52 +0800 Subject: [PATCH 20/43] fix(test): add test case. --- source/util/test/decompressTest.cpp | 94 +++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 source/util/test/decompressTest.cpp diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp new file mode 100644 index 0000000000..0c4f660002 --- /dev/null +++ b/source/util/test/decompressTest.cpp @@ -0,0 +1,94 @@ +#include +#include +#include +#include + +namespace {} // namespace + +TEST(utilTest, decompress_test) { + int64_t tsList[10] = {1700000000, 1700000100, 1700000200, 1700000300, 1700000400, + 1700000500, 1700000600, 1700000700, 1700000800, 1700000900}; + + char* pOutput[10 * sizeof(int64_t)] = {0}; + int32_t len = tsCompressTimestamp(tsList, sizeof(tsList), sizeof(tsList) / sizeof(tsList[0]), pOutput, 10, ONE_STAGE_COMP, NULL, 0); + + char* decompOutput[10 * 8] = {0}; + tsDecompressTimestamp(pOutput, len, 10, decompOutput, sizeof(int64_t)*10, ONE_STAGE_COMP, NULL, 0); + + for(int32_t i = 0; i < 10; ++i) { + std::cout<< ((int64_t*)decompOutput)[i] << std::endl; + } + + memset(decompOutput, 0, 10*8); + tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 10, + reinterpret_cast(decompOutput), false); + + for(int32_t i = 0; i < 10; ++i) { + std::cout<<((int64_t*)decompOutput)[i] << std::endl; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int64_t tsList1[7] = {1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000000, 1700000900}; + int32_t len1 = tsCompressTimestamp(tsList1, sizeof(tsList1), sizeof(tsList1) / sizeof(tsList1[0]), pOutput, 7, ONE_STAGE_COMP, NULL, 0); + + memset(decompOutput, 0, 10*8); + tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 7, + reinterpret_cast(decompOutput), false); + + for(int32_t i = 0; i < 7; ++i) { + std::cout<<((int64_t*)decompOutput)[i] << std::endl; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int64_t tsList2[1] = {1700000000}; + int32_t len2 = tsCompressTimestamp(tsList2, sizeof(tsList2), sizeof(tsList2) / sizeof(tsList2[0]), pOutput, 1, ONE_STAGE_COMP, NULL, 0); + + memset(decompOutput, 0, 10*8); + tsDecompressTimestampAvx512(reinterpret_cast(pOutput), 1, + reinterpret_cast(decompOutput), false); + + for(int32_t i = 0; i < 1; ++i) { + std::cout<<((int64_t*)decompOutput)[i] << std::endl; + } +} + +TEST(utilTest, decompress_perf_test) { + int32_t num = 100000; + + int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); + int64_t iniVal = 1700000000; + + uint32_t v = 100; + + for(int32_t i = 0; i < num; ++i) { + iniVal += taosRandR(&v)%10; + pList[i] = iniVal; + } + + char* px = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); + int32_t len = tsCompressTimestamp(pList, num * sizeof(int64_t), num, px, num, ONE_STAGE_COMP, NULL, 0); + + char* pOutput = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); + + int64_t st = taosGetTimestampUs(); + for(int32_t k = 0; k < 10; ++k) { + tsDecompressTimestamp(px, len, num, pOutput, sizeof(int64_t) * num, ONE_STAGE_COMP, NULL, 0); + } + + int64_t el1 = taosGetTimestampUs() - st; + std::cout << "decompress elapsed time:" << el1 << " us" << std::endl; + + memset(pOutput, 0, num * sizeof(int64_t)); + st = taosGetTimestampUs(); + for(int32_t k = 0; k < 10; ++k) { + tsDecompressTimestampAvx512(px, num, pOutput, false); + } + + int64_t el2 = taosGetTimestampUs() - st; + std::cout << "SIMD decompress elapsed time:" << el2 << " us" << std::endl; + + taosMemoryFree(pList); + taosMemoryFree(pOutput); + taosMemoryFree(px); +} + From bda2cd306d7cf8fcd29f1fcdec1476103807041e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 27 Nov 2023 09:29:43 +0800 Subject: [PATCH 21/43] fix:memory leak & comment length --- source/client/src/clientMain.c | 1 + source/common/src/systable.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 2091bcf64f..3a91ae82d5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -876,6 +876,7 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { code = 0; } taosThreadMutexUnlock(&pTscObj->mutex); + releaseTscObj(*(int64_t *)taos); return code; } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 89995fc326..1623d9f062 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -140,7 +140,7 @@ static const SSysDbTableSchema userStbsSchema[] = { {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "watermark", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "max_delay", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "rollup", .bytes = 128 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, From 6ab14d39bbf63619e3c92eba8366acb5c20713c0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 09:32:47 +0800 Subject: [PATCH 22/43] fix(test): do some internal refactor. --- cmake/cmake.define | 19 +++++++++++++------ source/util/src/tdecompress.c | 11 ++++------- source/util/test/decompressTest.cpp | 6 +++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 3eb872cfee..7cc33b8252 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE FALSE) +set(CMAKE_VERBOSE_MAKEFILE TRUE) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory @@ -159,6 +159,7 @@ ELSE () CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) + CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) IF (COMPILER_SUPPORT_SSE42) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") @@ -166,11 +167,11 @@ ELSE () ENDIF() IF ("${SIMD_SUPPORT}" MATCHES "true") - IF (COMPILER_SUPPORT_FMA) + IF (COMPILER_SUPPORT_FMA) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma") - ENDIF() - IF (COMPILER_SUPPORT_AVX) + ENDIF() + IF (COMPILER_SUPPORT_AVX) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") ENDIF() @@ -182,8 +183,14 @@ ELSE () IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") - MESSAGE(STATUS "avx512 supported by gcc") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") + MESSAGE(STATUS "avx512f/avx512bmi supported by gcc") + ENDIF() + + IF (COMPILER_SUPPORT_AVX512VL) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl") + MESSAGE(STATUS "avx512vl supported by gcc") ENDIF() ENDIF() diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index b1c6df95c4..b39a340ac6 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -111,7 +111,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, __m256i signmask = _mm256_and_si256(_mm256_set1_epi64x(1), zigzagVal); signmask = _mm256_sub_epi64(_mm256_setzero_si256(), signmask); - // get the four zigzag values here + // get four zigzag values here __m256i delta = _mm256_xor_si256(_mm256_srli_epi64(zigzagVal, 1), signmask); // calculate the cumulative sum (prefix sum) for each number @@ -254,10 +254,11 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem __m128i prevDelta = _mm_setzero_si128(); // _mm_maskz_loadu_epi8 -#if __AVX512F__ +#if __AVX512VL__ int32_t batch = nelements >> 1; int32_t remainder = nelements & 0x01; + __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; int32_t i = 0; if (batch > 1) { @@ -267,7 +268,6 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); data2 = _mm_broadcastq_epi64(data2); @@ -305,7 +305,6 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); data2 = _mm_broadcastq_epi64(data2); @@ -357,10 +356,8 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem if (opos == 0) { ostream[opos++] = deltaOfDelta; } else { - int64_t prevV = prevVal[1]; - int64_t prevDeltaX = deltaOfDelta + prevDelta[1]; - ostream[opos++] = prevV + prevDeltaX; + ostream[opos++] = prevVal[1] + prevDeltaX; } } diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp index 0c4f660002..378b67edbb 100644 --- a/source/util/test/decompressTest.cpp +++ b/source/util/test/decompressTest.cpp @@ -53,7 +53,7 @@ TEST(utilTest, decompress_test) { } TEST(utilTest, decompress_perf_test) { - int32_t num = 100000; + int32_t num = 10000; int64_t* pList = static_cast(taosMemoryCalloc(num, sizeof(int64_t))); int64_t iniVal = 1700000000; @@ -71,7 +71,7 @@ TEST(utilTest, decompress_perf_test) { char* pOutput = static_cast(taosMemoryMalloc(num * sizeof(int64_t))); int64_t st = taosGetTimestampUs(); - for(int32_t k = 0; k < 10; ++k) { + for(int32_t k = 0; k < 10000; ++k) { tsDecompressTimestamp(px, len, num, pOutput, sizeof(int64_t) * num, ONE_STAGE_COMP, NULL, 0); } @@ -80,7 +80,7 @@ TEST(utilTest, decompress_perf_test) { memset(pOutput, 0, num * sizeof(int64_t)); st = taosGetTimestampUs(); - for(int32_t k = 0; k < 10; ++k) { + for(int32_t k = 0; k < 10000; ++k) { tsDecompressTimestampAvx512(px, num, pOutput, false); } From 5de59a354787b2f941bdd626238ecbde17592cc8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 10:32:47 +0800 Subject: [PATCH 23/43] enh(sys): add check for avx512vl --- cmake/cmake.define | 4 +- source/util/src/tcompression.c | 101 +++++++++++++++++---------------- 2 files changed, 53 insertions(+), 52 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 7cc33b8252..c685ba6161 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -184,13 +184,13 @@ ELSE () IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") - MESSAGE(STATUS "avx512f/avx512bmi supported by gcc") + MESSAGE(STATUS "avx512f/avx512bmi supported by compiler") ENDIF() IF (COMPILER_SUPPORT_AVX512VL) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl") - MESSAGE(STATUS "avx512vl supported by gcc") + MESSAGE(STATUS "avx512vl supported by compiler") ENDIF() ENDIF() diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 09f50d0f6d..8a79d62cda 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -538,65 +538,66 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement memcpy(output, input + 1, nelements * longBytes); return nelements * longBytes; } else if (input[0] == 1) { // Decompress - int64_t *ostream = (int64_t *)output; + if (tsSIMDEnable && tsAVX512Enable) { + tsDecompressTimestampAvx512(input, nelements, output, false); + } else { + int64_t *ostream = (int64_t *)output; - int32_t ipos = 1, opos = 0; - int8_t nbytes = 0; - int64_t prev_value = 0; - int64_t prev_delta = 0; - int64_t delta_of_delta = 0; + int32_t ipos = 1, opos = 0; + int8_t nbytes = 0; + int64_t prev_value = 0; + int64_t prev_delta = 0; + int64_t delta_of_delta = 0; - while (1) { - uint8_t flags = input[ipos++]; - // Decode dd1 - uint64_t dd1 = 0; - nbytes = flags & INT8MASK(4); - if (nbytes == 0) { - delta_of_delta = 0; - } else { - if (is_bigendian()) { - memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + while (1) { + uint8_t flags = input[ipos++]; + // Decode dd1 + uint64_t dd1 = 0; + nbytes = flags & INT8MASK(4); + if (nbytes == 0) { + delta_of_delta = 0; } else { - memcpy(&dd1, input + ipos, nbytes); + if (is_bigendian()) { + memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + } else { + memcpy(&dd1, input + ipos, nbytes); + } + delta_of_delta = ZIGZAG_DECODE(int64_t, dd1); } - delta_of_delta = ZIGZAG_DECODE(int64_t, dd1); - } - ipos += nbytes; - if (opos == 0) { - prev_value = delta_of_delta; - prev_delta = 0; - ostream[opos++] = delta_of_delta; - } else { + + ipos += nbytes; + if (opos == 0) { + prev_value = delta_of_delta; + prev_delta = 0; + ostream[opos++] = delta_of_delta; + } else { + prev_delta = delta_of_delta + prev_delta; + prev_value = prev_value + prev_delta; + ostream[opos++] = prev_value; + } + if (opos == nelements) return nelements * longBytes; + + // Decode dd2 + uint64_t dd2 = 0; + nbytes = (flags >> 4) & INT8MASK(4); + if (nbytes == 0) { + delta_of_delta = 0; + } else { + if (is_bigendian()) { + memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); + } else { + memcpy(&dd2, input + ipos, nbytes); + } + // zigzag_decoding + delta_of_delta = ZIGZAG_DECODE(int64_t, dd2); + } + ipos += nbytes; prev_delta = delta_of_delta + prev_delta; prev_value = prev_value + prev_delta; ostream[opos++] = prev_value; + if (opos == nelements) return nelements * longBytes; } - if (opos == nelements) return nelements * longBytes; - - // Decode dd2 - uint64_t dd2 = 0; - nbytes = (flags >> 4) & INT8MASK(4); - if (nbytes == 0) { - delta_of_delta = 0; - } else { - if (is_bigendian()) { - memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); - } else { - memcpy(&dd2, input + ipos, nbytes); - } - // zigzag_decoding - delta_of_delta = ZIGZAG_DECODE(int64_t, dd2); - } - ipos += nbytes; - prev_delta = delta_of_delta + prev_delta; - prev_value = prev_value + prev_delta; - ostream[opos++] = prev_value; - if (opos == nelements) return nelements * longBytes; } - - } else { - ASSERT(0); - return -1; } } From 2ddb246461255ada404d0f054a73385a3f68ddb9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Nov 2023 14:10:00 +0800 Subject: [PATCH 24/43] change debug info --- source/libs/transport/src/transCli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ef60c8a94e..772996548d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -957,8 +957,8 @@ static void cliSendCb(uv_write_t* req, int status) { SCliMsg* pMsg = !transQueueEmpty(&pConn->cliMsgs) ? transQueueGet(&pConn->cliMsgs, 0) : NULL; if (pMsg != NULL) { int64_t cost = taosGetTimestampUs() - pMsg->st; - if (cost > 1000 * 20) { - tWarn("%s conn %p send cost:%dus, send exception", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); + if (cost > 1000 * 50) { + tWarn("%s conn %p send cost:%dus ", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); } } From d4c3bcc9f746baf4c7a0a31c0753715551c404b9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Nov 2023 14:10:38 +0800 Subject: [PATCH 25/43] change debug info --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 772996548d..e51c61c49d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -958,7 +958,7 @@ static void cliSendCb(uv_write_t* req, int status) { if (pMsg != NULL) { int64_t cost = taosGetTimestampUs() - pMsg->st; if (cost > 1000 * 50) { - tWarn("%s conn %p send cost:%dus ", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); + tTrace("%s conn %p send cost:%dus ", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); } } From f9d717d0f2fdc7b0a9917a29831d20e6f021eabd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 14:42:13 +0800 Subject: [PATCH 26/43] refactor: do some internal refactor. --- cmake/cmake.define | 4 +- include/util/tcompression.h | 1 + source/util/src/tcompression.c | 4 + source/util/src/tdecompress.c | 165 ++++++++++++++++++++++++++-- source/util/test/decompressTest.cpp | 2 +- 5 files changed, 164 insertions(+), 12 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index c685ba6161..7db6baafab 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -182,8 +182,8 @@ ELSE () MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi -mavx512vl") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi") MESSAGE(STATUS "avx512f/avx512bmi supported by compiler") ENDIF() diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 7da3587f1c..75ddbb12e7 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -140,6 +140,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian); +int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelements, char *const output, bool bigEndian); /************************************************************************* * STREAM COMPRESSION diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 8a79d62cda..3cc00ddc7f 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -540,6 +540,8 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement } else if (input[0] == 1) { // Decompress if (tsSIMDEnable && tsAVX512Enable) { tsDecompressTimestampAvx512(input, nelements, output, false); + } else if (tsSIMDEnable && tsAVX2Enable) { + tsDecompressTimestampAvx2(input, nelements, output, false); } else { int64_t *ostream = (int64_t *)output; @@ -599,6 +601,8 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement } } } + + return nelements * longBytes; } /* --------------------------------------------Double Compression ---------------------------------------------- */ diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index b39a340ac6..5a5e60093c 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -247,15 +247,14 @@ int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelemen return 0; } -int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian) { +int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, + bool bigEndian) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; __m128i prevVal = _mm_setzero_si128(); __m128i prevDelta = _mm_setzero_si128(); - // _mm_maskz_loadu_epi8 -#if __AVX512VL__ - +#if __AVX2__ int32_t batch = nelements >> 1; int32_t remainder = nelements & 0x01; __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; @@ -268,10 +267,22 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); - __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); - data2 = _mm_broadcastq_epi64(data2); + __m128i data1; + if (nbytes1 == 0) { + data1 = _mm_setzero_si128(); + } else { +// _mm_shuffle_epi8() + memcpy(&data1, (const void*) (input + ipos), nbytes1); + } + __m128i data2; + if (nbytes2 == 0) { + data2 = _mm_setzero_si128(); + } else { + memcpy(&data2, (const void*) (input + ipos + nbytes1), nbytes2); + } + + data2 = _mm_broadcastq_epi64(data2); __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) @@ -305,8 +316,26 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); - __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); +// __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); +// __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + __m128i data1; + if (nbytes1 == 0) { + data1 = _mm_setzero_si128(); + } else { + int64_t dd = 0; + memcpy(&dd, (const void*) (input + ipos), nbytes1); + data1 = _mm_loadu_si64(&dd); + } + + __m128i data2; + if (nbytes2 == 0) { + data2 = _mm_setzero_si128(); + } else { + int64_t dd = 0; + memcpy(&dd, (const void*) (input + ipos + nbytes1), nbytes2); + data2 = _mm_loadu_si64(&dd); + } + data2 = _mm_broadcastq_epi64(data2); __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); @@ -335,6 +364,124 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem ipos += nbytes1 + nbytes2; } + if (remainder > 0) { + uint64_t dd = 0; + uint8_t flags = input[ipos++]; + + int32_t nbytes = flags & INT8MASK(4); + int64_t deltaOfDelta = 0; + if (nbytes == 0) { + deltaOfDelta = 0; + } else { + // if (is_bigendian()) { + // memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); + // } else { + memcpy(&dd, input + ipos, nbytes); + // } + deltaOfDelta = ZIGZAG_DECODE(int64_t, dd); + } + + ipos += nbytes; + if (opos == 0) { + ostream[opos++] = deltaOfDelta; + } else { + int64_t prevDeltaX = deltaOfDelta + prevDelta[1]; + ostream[opos++] = prevVal[1] + prevDeltaX; + } + } +#endif + + return 0; +} + +int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian) { + int64_t *ostream = (int64_t *)output; + int32_t ipos = 1, opos = 0; + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); + + // _mm_maskz_loadu_epi8 +#if __AVX512VL__ + + int32_t batch = nelements >> 1; + int32_t remainder = nelements & 0x01; + __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; + + int32_t i = 0; + if (batch > 1) { + // first loop + uint8_t flags = input[ipos++]; + + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + data2 = _mm_broadcastq_epi64(data2); + + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_shuffle_epi32 (val, 0xEE); + + // keep the previous delta of delta, for the first item + prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); + + opos += 2; + ipos += nbytes1 + nbytes2; + i += 1; + } + + // the remain + for(; i < batch; ++i) { + uint8_t flags = input[ipos++]; + + int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 + int8_t nbytes2 = (flags >> 4) & INT8MASK(4); + + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + data2 = _mm_broadcastq_epi64(data2); + + __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m128i signmask = _mm_and_si128(_mm_set1_epi64x(1), zzVal); + signmask = _mm_sub_epi64(_mm_setzero_si128(), signmask); + + // get two zigzag values here + __m128i deltaOfDelta = _mm_xor_si128(_mm_srli_epi64(zzVal, 1), signmask); + + __m128i deltaCurrent = _mm_add_epi64(deltaOfDelta, prevDelta); + deltaCurrent = _mm_add_epi64(_mm_slli_si128(deltaCurrent, 8), deltaCurrent); + + __m128i val = _mm_add_epi64(deltaCurrent, prevVal); + _mm_storeu_si128((__m128i *)&ostream[opos], val); + + // keep the previous value + prevVal = _mm_shuffle_epi32 (val, 0xEE); + + // keep the previous delta of delta + __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta); + prevDelta = _mm_shuffle_epi32(_mm_add_epi64(delta, prevDelta), 0xEE); + + opos += 2; + ipos += nbytes1 + nbytes2; + } + if (remainder > 0) { uint64_t dd = 0; uint8_t flags = input[ipos++]; diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp index 378b67edbb..caf8df3ba8 100644 --- a/source/util/test/decompressTest.cpp +++ b/source/util/test/decompressTest.cpp @@ -76,7 +76,7 @@ TEST(utilTest, decompress_perf_test) { } int64_t el1 = taosGetTimestampUs() - st; - std::cout << "decompress elapsed time:" << el1 << " us" << std::endl; + std::cout << "soft decompress elapsed time:" << el1 << " us" << std::endl; memset(pOutput, 0, num * sizeof(int64_t)); st = taosGetTimestampUs(); From d1193b5a7a29d3df3811d77e6bdd5d5ec84f8721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 14:59:12 +0800 Subject: [PATCH 27/43] refactor: opt simd perf. --- source/util/src/tdecompress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 5a5e60093c..509afb6ed4 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -299,10 +299,10 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen _mm_storeu_si128((__m128i *)&ostream[opos], val); // keep the previous value - prevVal = _mm_set1_epi64x(val[1]); + prevVal = _mm_shuffle_epi32 (val, 0xEE); // keep the previous delta of delta, for the first item - prevDelta = _mm_set1_epi64x(deltaOfDelta[1]); + prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); opos += 2; ipos += nbytes1 + nbytes2; @@ -354,11 +354,11 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen _mm_storeu_si128((__m128i *)&ostream[opos], val); // keep the previous value - prevVal = _mm_set1_epi64x(val[1]); + prevVal = _mm_shuffle_epi32 (val, 0xEE); // keep the previous delta of delta __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta); - prevDelta = _mm_set1_epi64x(_mm_add_epi64(delta, prevDelta)[1]); + prevDelta = _mm_shuffle_epi32(_mm_add_epi64(delta, prevDelta), 0xEE); opos += 2; ipos += nbytes1 + nbytes2; From 61111385676a910cc9b956fc111c7858da2322f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 17:23:34 +0800 Subject: [PATCH 28/43] fix(stream): fix error. --- source/libs/stream/src/streamExec.c | 18 +++++++++--------- source/util/src/tdecompress.c | 10 ++++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 8b14846414..e0ee01d345 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -252,14 +252,15 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.pExecutor; - bool finished = false; + void* exec = pTask->exec.pExecutor; + bool finished = false; + const char* id = pTask->id.idStr; qSetStreamOpOpen(exec); while (1) { if (streamTaskShouldPause(pTask)) { - stDebug("s-task:%s paused from the scan-history task", pTask->id.idStr); + stDebug("s-task:%s paused from the scan-history task", id); // quit from step1, not continue to handle the step2 return (SScanhistoryDataInfo){TASK_SCANHISTORY_QUIT, 0}; } @@ -267,8 +268,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - stError("s-task:%s scan-history prepare result block failed, code:%s, retry later", pTask->id.idStr, - tstrerror(terrno)); + stError("s-task:%s scan-history prepare result block failed, code:%s, retry later", id, tstrerror(terrno)); continue; } @@ -295,8 +295,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { } if (el >= STREAM_SCAN_HISTORY_TIMESLICE) { - stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", - pTask->id.idStr, pTask->info.fillHistory, el / 1000.0); + stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id, + pTask->info.fillHistory, el / 1000.0); return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 100}; } } @@ -543,7 +543,7 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the * appropriate batch of blocks should be handled in 5 to 10 sec. */ -int32_t streamExecForAll(SStreamTask* pTask) { +int32_t doStreamExecTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; // merge multiple input data if possible in the input queue. @@ -654,7 +654,7 @@ int32_t streamExecTask(SStreamTask* pTask) { int8_t schedStatus = streamTaskSetSchedStatusActive(pTask); if (schedStatus == TASK_SCHED_STATUS__WAITING) { while (1) { - int32_t code = streamExecForAll(pTask); + int32_t code = doStreamExecTask(pTask); if (code < 0) { // todo this status should be removed atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 509afb6ed4..bb22772f7d 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -271,7 +271,6 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen if (nbytes1 == 0) { data1 = _mm_setzero_si128(); } else { -// _mm_shuffle_epi8() memcpy(&data1, (const void*) (input + ipos), nbytes1); } @@ -400,15 +399,14 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem __m128i prevVal = _mm_setzero_si128(); __m128i prevDelta = _mm_setzero_si128(); - // _mm_maskz_loadu_epi8 #if __AVX512VL__ - int32_t batch = nelements >> 1; - int32_t remainder = nelements & 0x01; + int32_t numOfBatch = nelements >> 1; + int32_t remainder = nelements & 0x01; __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; int32_t i = 0; - if (batch > 1) { + if (numOfBatch > 1) { // first loop uint8_t flags = input[ipos++]; @@ -446,7 +444,7 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem } // the remain - for(; i < batch; ++i) { + for(; i < numOfBatch; ++i) { uint8_t flags = input[ipos++]; int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 From 70c5d361031436e07ea4db5fbe7725315a86bf86 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 27 Nov 2023 17:49:09 +0800 Subject: [PATCH 29/43] opti:extract common logic from snode & vnode to tqCommon --- include/dnode/vnode/tqCommon.h | 36 + include/libs/stream/tstream.h | 17 - source/dnode/mnode/impl/src/mndStream.c | 13 +- source/dnode/snode/CMakeLists.txt | 2 + source/dnode/snode/src/snode.c | 26 +- source/dnode/vnode/CMakeLists.txt | 4 + source/dnode/vnode/src/tq/tq.c | 25 +- .../dnode/vnode/src/tqCommon/CMakeLists.txt | 20 + source/dnode/vnode/src/tqCommon/tqCommon.c | 809 ++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSync.c | 3 +- source/libs/stream/inc/streamInt.h | 6 - source/libs/stream/src/streamTask.c | 782 ----------------- 12 files changed, 910 insertions(+), 833 deletions(-) create mode 100644 include/dnode/vnode/tqCommon.h create mode 100644 source/dnode/vnode/src/tqCommon/CMakeLists.txt create mode 100644 source/dnode/vnode/src/tqCommon/tqCommon.c diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h new file mode 100644 index 0000000000..75dafcdbff --- /dev/null +++ b/include/dnode/vnode/tqCommon.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TQ_COMMON_H +#define TDENGINE_TQ_COMMON_H + +// message process +int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart); +int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored); +int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); +int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); +int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); +int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); +int32_t startStreamTasks(SStreamMeta* pMeta); +int32_t resetStreamTaskStatus(SStreamMeta* pMeta); + +#endif // TDENGINE_TQ_COMMON_H diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index c187127fff..f8ac360c71 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -853,23 +853,6 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int8_t isSucceed); -// message process -int32_t streamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart); -int32_t streamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored); -int32_t streamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); -int32_t streamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t streamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); -int32_t streamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); -int32_t streamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); -int32_t startStreamTasks(SStreamMeta* pMeta); -int32_t resetStreamTaskStatus(SStreamMeta* pMeta); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c14d99ca36..0167c1a6ac 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -698,8 +698,17 @@ static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask STransAction action = {0}; SEpSet epset = {0}; if(pTask->info.nodeId == SNODE_HANDLE){ - SSnodeObj* pObj = mndAcquireSnode(pMnode, pTask->info.nodeId); - addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + SSnodeObj *pObj = NULL; + void *pIter = NULL; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + sdbRelease(pMnode->pSdb, pObj); + } }else{ SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); epset = mndGetVgroupEpset(pMnode, pVgObj); diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index ebfe80ecab..4cd8e26e78 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(snode STATIC ${SNODE_SRC}) target_include_directories( snode PUBLIC "${TD_SOURCE_DIR}/include/dnode/snode" + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( @@ -16,4 +17,5 @@ target_link_libraries( PRIVATE stream PRIVATE wal PRIVATE index + PRIVATE tqCommon ) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 8db2e783db..72ef6d14bd 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -16,7 +16,7 @@ #include "rsync.h" #include "executor.h" #include "sndInt.h" -#include "tstream.h" +#include "tqCommon.h" #include "tuuid.h" #define sndError(...) \ @@ -165,25 +165,25 @@ void sndClose(SSnode *pSnode) { int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: - return streamTaskProcessRunReq(pSnode->pMeta, pMsg, true); + return tqStreamTaskProcessRunReq(pSnode->pMeta, pMsg, true); case TDMT_STREAM_TASK_DISPATCH: - return streamTaskProcessDispatchReq(pSnode->pMeta, pMsg); + return tqStreamTaskProcessDispatchReq(pSnode->pMeta, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: - return streamTaskProcessDispatchRsp(pSnode->pMeta, pMsg); + return tqStreamTaskProcessDispatchRsp(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE: - return streamTaskProcessRetrieveReq(pSnode->pMeta, pMsg); + return tqStreamTaskProcessRetrieveReq(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE_RSP: // 1036 break; case TDMT_VND_STREAM_SCAN_HISTORY_FINISH: - return streamTaskProcessScanHistoryFinishReq(pSnode->pMeta, pMsg); + return tqStreamTaskProcessScanHistoryFinishReq(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP: - return streamTaskProcessScanHistoryFinishRsp(pSnode->pMeta, pMsg); + return tqStreamTaskProcessScanHistoryFinishRsp(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_TASK_CHECK: - return streamTaskProcessCheckReq(pSnode->pMeta, pMsg); + return tqStreamTaskProcessCheckReq(pSnode->pMeta, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: - return streamTaskProcessCheckRsp(pSnode->pMeta, pMsg, true); + return tqStreamTaskProcessCheckRsp(pSnode->pMeta, pMsg, true); case TDMT_STREAM_TASK_CHECKPOINT_READY: - return streamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg); + return tqStreamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg); default: sndError("invalid snode msg:%d", pMsg->msgType); ASSERT(0); @@ -196,13 +196,13 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DEPLOY: { void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return streamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); + return tqStreamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); } case TDMT_STREAM_TASK_DROP: - return streamTaskProcessDropReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); + return tqStreamTaskProcessDropReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); case TDMT_VND_STREAM_TASK_UPDATE: - return streamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); + return tqStreamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); default: ASSERT(0); } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dc43da7fe7..9aeb14cd60 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,4 +1,5 @@ # vnode +add_subdirectory(src/tqCommon) add_library(vnode STATIC "") set( VNODE_SOURCE_FILES @@ -117,6 +118,7 @@ if (${BUILD_CONTRIB}) PUBLIC "inc" PUBLIC "src/inc" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" ) else() @@ -125,6 +127,7 @@ else() PUBLIC "inc" PUBLIC "src/inc" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" ) if (${TD_LINUX}) target_include_directories( @@ -158,6 +161,7 @@ target_link_libraries( PUBLIC transport PUBLIC stream PUBLIC index + PUBLIC tqCommon ) IF (TD_GRANT) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1145fa328d..5b04b31c94 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,6 +15,7 @@ #include "tq.h" #include "vnd.h" +#include "tqCommon.h" typedef struct { int8_t inited; @@ -890,15 +891,15 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { } int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessCheckReq(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessCheckReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessCheckRsp(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); + return tqStreamTaskProcessCheckRsp(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); } int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - return streamTaskProcessDeployReq(pTq->pStreamMeta, sversion, msg, msgLen, vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored); + return tqStreamTaskProcessDeployReq(pTq->pStreamMeta, sversion, msg, msgLen, vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored); } static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { @@ -1080,11 +1081,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // only the agg tasks and the sink tasks will receive this message from upstream tasks int32_t tqProcessTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessScanHistoryFinishReq(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessScanHistoryFinishReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessScanHistoryFinishRsp(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessScanHistoryFinishRsp(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { @@ -1096,7 +1097,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { tqScanWal(pTq); return 0; } - int32_t code = streamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); + int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); if(code == 0 && taskId > 0){ tqScanWalAsync(pTq, false); } @@ -1104,15 +1105,15 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessDispatchRsp(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessDispatchRsp(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { - return streamTaskProcessDropReq(pTq->pStreamMeta, msg, msgLen); + return tqStreamTaskProcessDropReq(pTq->pStreamMeta, msg, msgLen); } int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { @@ -1221,7 +1222,7 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms } int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessRetrieveReq(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessRetrieveReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) { @@ -1358,11 +1359,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessCheckpointReadyMsg(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessCheckpointReadyMsg(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { - return streamTaskProcessUpdateReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, pMsg, pTq->pVnode->restored); + return tqStreamTaskProcessUpdateReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, pMsg, pTq->pVnode->restored); } int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tqCommon/CMakeLists.txt b/source/dnode/vnode/src/tqCommon/CMakeLists.txt new file mode 100644 index 0000000000..aea0e709e3 --- /dev/null +++ b/source/dnode/vnode/src/tqCommon/CMakeLists.txt @@ -0,0 +1,20 @@ +aux_source_directory(. TQ_SOURCE_FILES) +add_library(tqCommon STATIC ${TQ_SOURCE_FILES}) +target_include_directories( + tqCommon + PUBLIC "../inc" + PUBLIC "../../inc" +) + +target_link_libraries( + tqCommon + PRIVATE stream + PRIVATE common + PRIVATE transport + PRIVATE executor + PRIVATE index + PRIVATE qcom + PRIVATE qworker + PRIVATE sync + PRIVATE tfs +) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c new file mode 100644 index 0000000000..aee2aaa244 --- /dev/null +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -0,0 +1,809 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" +#include "tmsgcb.h" +#include "tq.h" + +typedef struct STaskUpdateEntry { + int64_t streamId; + int32_t taskId; + int32_t transId; +} STaskUpdateEntry; + +int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + return -1; + } + + tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(cb, STREAM_QUEUE, &msg); + return 0; +} + +int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { + int32_t vgId = pMeta->vgId; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; + + SStreamTaskNodeUpdateMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { + rsp.code = TSDB_CODE_MSG_DECODE_ERROR; + tqError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); + tDecoderClear(&decoder); + return rsp.code; + } + + tDecoderClear(&decoder); + + // update the nodeEpset when it exists + streamMetaWLock(pMeta); + + // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. + STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask == NULL || *ppTask == NULL) { + tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, + req.taskId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + SStreamTask* pTask = *ppTask; + + if (pMeta->updateInfo.transId != req.transId) { + pMeta->updateInfo.transId = req.transId; + tqInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); + // info needs to be kept till the new trans to update the nodeEp arrived. + taosHashClear(pMeta->updateInfo.pTasks); + } else { + tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); + } + + STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; + void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); + if (exist != NULL) { + tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, + req.transId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + streamMetaWUnLock(pMeta); + + // the following two functions should not be executed within the scope of meta lock to avoid deadlock + streamTaskUpdateEpsetInfo(pTask, req.pNodeList); + streamTaskResetStatus(pTask); + + // continue after lock the meta again + streamMetaWLock(pMeta); + + SStreamTask** ppHTask = NULL; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); + if (ppHTask == NULL || *ppHTask == NULL) { + tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", + pMeta->vgId, req.taskId); + CLEAR_RELATED_FILLHISTORY_TASK(pTask); + } else { + tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); + streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); + } + } + + { + streamMetaSaveTask(pMeta, pTask); + if (ppHTask != NULL) { + streamMetaSaveTask(pMeta, *ppHTask); + } + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + } + + streamTaskStop(pTask); + + // keep the already handled info + taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); + + if (ppHTask != NULL) { + streamTaskStop(*ppHTask); + tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); + taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); + } else { + tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); + } + + rsp.code = 0; + + // possibly only handle the stream task. + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); + + pMeta->startInfo.tasksWillRestart = 1; + + if (updateTasks < numOfTasks) { + tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, + updateTasks, (numOfTasks - updateTasks)); + streamMetaWUnLock(pMeta); + } else { + if (!restored) { + tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); + pMeta->startInfo.tasksWillRestart = 0; + streamMetaWUnLock(pMeta); + } else { + tqDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); +#if 1 + tqStreamTaskStartAsync(pMeta, cb, true); + streamMetaWUnLock(pMeta); +#else + streamMetaWUnLock(pMeta); + + // For debug purpose. + // the following procedure consume many CPU resource, result in the re-election of leader + // with high probability. So we employ it as a test case for the stream processing framework, with + // checkpoint/restart/nodeUpdate etc. + while (1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + while (streamMetaTaskInTimer(pMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + int32_t code = streamMetaReopen(pMeta); + if (code != 0) { + tqError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + streamMetaInitBackend(pMeta); + + if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { + tqError("vgId:%d failed to load stream tasks", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { + tqInfo("vgId:%d start all stream tasks after all being updated", vgId); + resetStreamTaskStatus(pTq->pStreamMeta); + tqStartStreamTaskAsync(pTq, false); + } else { + tqInfo("vgId:%d, follower node not start stream tasks", vgId); + } + streamMetaWUnLock(pMeta); +#endif + } + } + + taosArrayDestroy(req.pNodeList); + return rsp.code; +} + +int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamDispatchReq req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); + if (tDecodeStreamDispatchReq(&decoder, &req) < 0) { + tDecoderClear(&decoder); + return TSDB_CODE_MSG_DECODE_ERROR; + } + tDecoderClear(&decoder); + + tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); + if (pTask) { + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; + if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0){ + return -1; + } + tDeleteStreamDispatchReq(&req); + streamMetaReleaseTask(pMeta, pTask); + return 0; + } else { + tqError("vgId:%d failed to find task:0x%x to handle the dispatch req, it may have been destroyed already", + pMeta->vgId, req.taskId); + + SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); + if (pRspHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("s-task:0x%x send dispatch error rsp, out of memory", req.taskId); + return -1; + } + + pRspHead->vgId = htonl(req.upstreamNodeId); + ASSERT(pRspHead->vgId != 0); + + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead)); + pRsp->streamId = htobe64(req.streamId); + pRsp->upstreamTaskId = htonl(req.upstreamTaskId); + pRsp->upstreamNodeId = htonl(req.upstreamNodeId); + pRsp->downstreamNodeId = htonl(pMeta->vgId); + pRsp->downstreamTaskId = htonl(req.taskId); + pRsp->msgId = htonl(req.msgId); + pRsp->stage = htobe64(req.stage); + pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; + + int32_t len = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); + SRpcMsg rsp = {.code = TSDB_CODE_STREAM_TASK_NOT_EXIST, .info = pMsg->info, .contLen = len, .pCont = pRspHead}; + tqError("s-task:0x%x send dispatch error rsp, no task", req.taskId); + + tmsgSendRsp(&rsp); + tDeleteStreamDispatchReq(&req); + + return 0; + } +} + +int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + + int32_t vgId = pMeta->vgId; + pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); + pRsp->streamId = htobe64(pRsp->streamId); + pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); + pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); + pRsp->stage = htobe64(pRsp->stage); + pRsp->msgId = htonl(pRsp->msgId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); + if (pTask) { + streamProcessDispatchRsp(pTask, pRsp, pMsg->code); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; + } else { + tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); + terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; + return terrno; + } +} + +int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder; + + SStreamRetrieveReq req; + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); + tDecodeStreamRetrieveReq(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId); + if (pTask == NULL) { + tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, + req.dstTaskId); + return -1; + } + + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; + streamProcessRetrieveReq(pTask, &req, &rsp); + + streamMetaReleaseTask(pMeta, pTask); + tDeleteStreamRetrieveReq(&req); + return 0; +} + +int32_t tqStreamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamScanHistoryFinishReq req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.downstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", + pMeta->vgId, req.downstreamTaskId); + return -1; + } + + tqDebug("s-task:%s receive scan-history finish msg from task:0x%x", pTask->id.idStr, req.upstreamTaskId); + + int32_t code = streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + +int32_t tqStreamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamCompleteHistoryMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeCompleteHistoryDataMsg(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", + pMeta->vgId, req.upstreamTaskId); + return -1; + } + + int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); + if (remain > 0) { + tqDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", + pTask->id.idStr, req.downstreamId, remain); + } else { + tqDebug( + "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " + "completed msg", + pTask->id.idStr, req.downstreamId); + streamProcessScanHistoryFinishRsp(pTask); + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + +int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamTaskCheckReq req; + SDecoder decoder; + + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); + tDecodeStreamTaskCheckReq(&decoder, &req); + tDecoderClear(&decoder); + + int32_t taskId = req.downstreamTaskId; + + SStreamTaskCheckRsp rsp = { + .reqId = req.reqId, + .streamId = req.streamId, + .childId = req.childId, + .downstreamNodeId = req.downstreamNodeId, + .downstreamTaskId = req.downstreamTaskId, + .upstreamNodeId = req.upstreamNodeId, + .upstreamTaskId = req.upstreamTaskId, + }; + + // only the leader node handle the check request + if (pMeta->role == NODE_ROLE_FOLLOWER) { + tqError("s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", + taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); + rsp.status = TASK_DOWNSTREAM_NOT_LEADER; + } else { + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); + if (pTask != NULL) { + rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); + streamMetaReleaseTask(pMeta, pTask); + + char* p = NULL; + streamTaskGetStatus(pTask, &p); + tqDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", + pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } else { + rsp.status = TASK_DOWNSTREAM_NOT_READY; + tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 + ") from task:0x%x (vgId:%d), rsp check_status %d", + req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } + } + + return streamSendCheckRsp(pMeta, &req, &rsp, &pMsg->info, taskId); +} + +int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t vgId = pMeta->vgId; + + int32_t code; + SStreamTaskCheckRsp rsp; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pReq, len); + code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); + if (code < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&decoder); + tqError("vgId:%d failed to parse check rsp msg, code:%s", vgId, tstrerror(terrno)); + return -1; + } + + tDecoderClear(&decoder); + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + + if (!isLeader) { + tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, + rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); + return code; + } + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); + if (pTask == NULL) { + tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", + rsp.streamId, rsp.upstreamTaskId, vgId); + terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; + return -1; + } + + code = streamProcessCheckRsp(pTask, &rsp); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + +int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { + int32_t vgId = pMeta->vgId; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; + + SStreamCheckpointReadyMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + tDecoderClear(&decoder); + return code; + } + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); + return code; + } + + tqDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId, + pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); + + streamProcessCheckpointReadyMsg(pTask); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + +int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored) { + int32_t code = 0; + int32_t vgId = pMeta->vgId; + + if (tsDisableStream) { + tqInfo("vgId:%d stream disabled, not deploy stream tasks", vgId); + return code; + } + + tqDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId); + + // 1.deserialize msg and build task + int32_t size = sizeof(SStreamTask); + SStreamTask* pTask = taosMemoryCalloc(1, size); + if (pTask == NULL) { + tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + code = tDecodeStreamTask(&decoder, pTask); + tDecoderClear(&decoder); + + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pTask); + return TSDB_CODE_INVALID_MSG; + } + + // 2.save task, use the latest commit version as the initial start version of stream task. + int32_t taskId = pTask->id.taskId; + int64_t streamId = pTask->id.streamId; + bool added = false; + + streamMetaWLock(pMeta); + code = streamMetaRegisterTask(pMeta, sversion, pTask, &added); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + streamMetaWUnLock(pMeta); + + if (code < 0) { + tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); + tFreeStreamTask(pTask); + return code; + } + + // added into meta store, pTask cannot be reference since it may have been destroyed by other threads already now if + // it is added into the meta store + if (added) { + // only handled in the leader node + if (isLeader) { + tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); + SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); + + if (p != NULL && restored && p->info.fillHistory == 0) { + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + streamTaskHandleEvent(p->status.pSM, event); + } else if (!restored) { + tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); + } + + if (p != NULL) { + streamMetaReleaseTask(pMeta, p); + } + } else { + tqDebug("vgId:%d not leader, not launch stream task s-task:0x%x", vgId, taskId); + } + } else { + tqWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store", vgId, taskId); + tFreeStreamTask(pTask); + } + + return code; +} + +int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { + SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + + int32_t vgId = pMeta->vgId; + tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask != NULL) { + // drop the related fill-history task firstly + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pHTaskId = &pTask->hTaskInfo.id; + streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); + tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId); + } + streamMetaReleaseTask(pMeta, pTask); + } + + // drop the stream task now + streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); + + // commit the update + streamMetaWLock(pMeta); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + tqDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks); + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + streamMetaWUnLock(pMeta); + + return 0; +} + +int32_t startStreamTasks(SStreamMeta* pMeta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + tqDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.startTs = taosGetTimestampMs(); + streamMetaWUnLock(pMeta); + + // broadcast the check downstream tasks msg + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + // fill-history task can only be launched by related stream tasks. + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + if (pTask->status.downstreamReady == 1) { + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", + pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return code; +} + +int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + + tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + + STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; + SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + streamTaskResetStatus(*pTask); + } + + return 0; +} + +static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { + int32_t vgId = pMeta->vgId; + int32_t code = 0; + int64_t st = taosGetTimestampMs(); + + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + terrno = 0; + tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, + pMeta->updateInfo.transId); + + while (streamMetaTaskInTimer(pMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + code = streamMetaReopen(pMeta); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + streamMetaInitBackend(pMeta); + int64_t el = taosGetTimestampMs() - st; + + tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); + + code = streamMetaLoadAllTasks(pMeta); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + if (isLeader && !tsDisableStream) { + tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + resetStreamTaskStatus(pMeta); + + streamMetaWUnLock(pMeta); + startStreamTasks(pMeta); + } else { + streamMetaResetStartInfo(&pMeta->startInfo); + streamMetaWUnLock(pMeta); + tqInfo("vgId:%d, follower node not start stream tasks", vgId); + } + + code = terrno; + return code; +} + +int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { + SStreamTaskRunReq* pReq = pMsg->pCont; + + int32_t taskId = pReq->taskId; + int32_t vgId = pMeta->vgId; + + if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { + startStreamTasks(pMeta); + return 0; + } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { + restartStreamTasks(pMeta, isLeader); + return 0; + } + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); + if (pTask != NULL) { // even in halt status, the data in inputQ must be processed + char* p = NULL; + if (streamTaskReadyToRun(pTask, &p)) { + tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, + pTask->chkInfo.nextProcessVer); + streamExecTask(pTask); + } else { + int8_t status = streamTaskSetSchedStatusInactive(pTask); + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, + pTask->id.idStr, p, status); + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; + } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. + // todo add one function to handle this + tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, taskId); + return -1; + } +} + + diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 1f0f8f7362..817d5124a2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -18,6 +18,7 @@ #include "sync.h" #include "tsdb.h" #include "vnd.h" +#include "tqCommon.h" #define BATCH_ENABLE 0 @@ -570,7 +571,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); resetStreamTaskStatus(pVnode->pTq->pStreamMeta); - streamTaskStartAsync(pMeta, &pVnode->msgCb, false); + tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); } } else { vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 60e64f31bb..4800e62109 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -52,12 +52,6 @@ extern "C" { #define stTrace(...) do { if (stDebugFlag & DEBUG_TRACE) { taosPrintLog("STM ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -typedef struct STaskUpdateEntry { - int64_t streamId; - int32_t taskId; - int32_t transId; -} STaskUpdateEntry; - typedef struct { int8_t type; SSDataBlock* pBlock; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 91d97a3068..69a8d1309a 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -784,785 +784,3 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->checkpointFailed = pSrc->checkpointFailed; } -int32_t streamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { - int32_t vgId = pMeta->vgId; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - stDebug("vgId:%d no stream tasks existed to run", vgId); - return 0; - } - - SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - stError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - return -1; - } - - stDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); - pRunReq->head.vgId = vgId; - pRunReq->streamId = 0; - pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - tmsgPutToQueue(cb, STREAM_QUEUE, &msg); - return 0; -} - -int32_t streamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { - int32_t vgId = pMeta->vgId; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; - - SStreamTaskNodeUpdateMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { - rsp.code = TSDB_CODE_MSG_DECODE_ERROR; - stError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); - tDecoderClear(&decoder); - return rsp.code; - } - - tDecoderClear(&decoder); - - // update the nodeEpset when it exists - streamMetaWLock(pMeta); - - // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. - STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - if (ppTask == NULL || *ppTask == NULL) { - stError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, - req.taskId); - rsp.code = TSDB_CODE_SUCCESS; - streamMetaWUnLock(pMeta); - - taosArrayDestroy(req.pNodeList); - return rsp.code; - } - - SStreamTask* pTask = *ppTask; - - if (pMeta->updateInfo.transId != req.transId) { - pMeta->updateInfo.transId = req.transId; - stInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); - // info needs to be kept till the new trans to update the nodeEp arrived. - taosHashClear(pMeta->updateInfo.pTasks); - } else { - stDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); - } - - STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; - void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); - if (exist != NULL) { - stDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, - req.transId); - rsp.code = TSDB_CODE_SUCCESS; - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return rsp.code; - } - - streamMetaWUnLock(pMeta); - - // the following two functions should not be executed within the scope of meta lock to avoid deadlock - streamTaskUpdateEpsetInfo(pTask, req.pNodeList); - streamTaskResetStatus(pTask); - - // continue after lock the meta again - streamMetaWLock(pMeta); - - SStreamTask** ppHTask = NULL; - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); - if (ppHTask == NULL || *ppHTask == NULL) { - stError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - pMeta->vgId, req.taskId); - CLEAR_RELATED_FILLHISTORY_TASK(pTask); - } else { - stDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); - streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); - } - } - - { - streamMetaSaveTask(pMeta, pTask); - if (ppHTask != NULL) { - streamMetaSaveTask(pMeta, *ppHTask); - } - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } - } - - streamTaskStop(pTask); - - // keep the already handled info - taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); - - if (ppHTask != NULL) { - streamTaskStop(*ppHTask); - stDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); - taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); - } else { - stDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); - } - - rsp.code = 0; - - // possibly only handle the stream task. - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); - - pMeta->startInfo.tasksWillRestart = 1; - - if (updateTasks < numOfTasks) { - stDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, - updateTasks, (numOfTasks - updateTasks)); - streamMetaWUnLock(pMeta); - } else { - if (!restored) { - stDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); - pMeta->startInfo.tasksWillRestart = 0; - streamMetaWUnLock(pMeta); - } else { - stDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); -#if 1 - streamTaskStartAsync(pMeta, cb, true); - streamMetaWUnLock(pMeta); -#else - streamMetaWUnLock(pMeta); - - // For debug purpose. - // the following procedure consume many CPU resource, result in the re-election of leader - // with high probability. So we employ it as a test case for the stream processing framework, with - // checkpoint/restart/nodeUpdate etc. - while (1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - int32_t code = streamMetaReopen(pMeta); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - streamMetaInitBackend(pMeta); - - if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { - tqError("vgId:%d failed to load stream tasks", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - resetStreamTaskStatus(pTq->pStreamMeta); - tqStartStreamTaskAsync(pTq, false); - } else { - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - streamMetaWUnLock(pMeta); -#endif - } - } - - taosArrayDestroy(req.pNodeList); - return rsp.code; -} - -int32_t streamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - SStreamDispatchReq req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - if (tDecodeStreamDispatchReq(&decoder, &req) < 0) { - tDecoderClear(&decoder); - return TSDB_CODE_MSG_DECODE_ERROR; - } - tDecoderClear(&decoder); - - stDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); - if (pTask) { - SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0){ - return -1; - } - tDeleteStreamDispatchReq(&req); - streamMetaReleaseTask(pMeta, pTask); - return 0; - } else { - stError("vgId:%d failed to find task:0x%x to handle the dispatch req, it may have been destroyed already", - pMeta->vgId, req.taskId); - - SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); - if (pRspHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - stError("s-task:0x%x send dispatch error rsp, out of memory", req.taskId); - return -1; - } - - pRspHead->vgId = htonl(req.upstreamNodeId); - ASSERT(pRspHead->vgId != 0); - - SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead)); - pRsp->streamId = htobe64(req.streamId); - pRsp->upstreamTaskId = htonl(req.upstreamTaskId); - pRsp->upstreamNodeId = htonl(req.upstreamNodeId); - pRsp->downstreamNodeId = htonl(pMeta->vgId); - pRsp->downstreamTaskId = htonl(req.taskId); - pRsp->msgId = htonl(req.msgId); - pRsp->stage = htobe64(req.stage); - pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; - - int32_t len = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); - SRpcMsg rsp = {.code = TSDB_CODE_STREAM_TASK_NOT_EXIST, .info = pMsg->info, .contLen = len, .pCont = pRspHead}; - stError("s-task:0x%x send dispatch error rsp, no task", req.taskId); - - tmsgSendRsp(&rsp); - tDeleteStreamDispatchReq(&req); - - return 0; - } -} - -int32_t streamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { - SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - - int32_t vgId = pMeta->vgId; - pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); - pRsp->streamId = htobe64(pRsp->streamId); - pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); - pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); - pRsp->stage = htobe64(pRsp->stage); - pRsp->msgId = htonl(pRsp->msgId); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); - if (pTask) { - streamProcessDispatchRsp(pTask, pRsp, pMsg->code); - streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; - } else { - stDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); - terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; - return terrno; - } -} - -int32_t streamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - SDecoder decoder; - - SStreamRetrieveReq req; - tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamRetrieveReq(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId); - if (pTask == NULL) { - stError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, - req.dstTaskId); - return -1; - } - - SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessRetrieveReq(pTask, &req, &rsp); - - streamMetaReleaseTask(pMeta, pTask); - tDeleteStreamRetrieveReq(&req); - return 0; -} - -int32_t streamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - // deserialize - SStreamScanHistoryFinishReq req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeStreamScanHistoryFinishReq(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.downstreamTaskId); - if (pTask == NULL) { - stError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", - pMeta->vgId, req.downstreamTaskId); - return -1; - } - - stDebug("s-task:%s receive scan-history finish msg from task:0x%x", pTask->id.idStr, req.upstreamTaskId); - - int32_t code = streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info); - streamMetaReleaseTask(pMeta, pTask); - return code; -} - -int32_t streamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - // deserialize - SStreamCompleteHistoryMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeCompleteHistoryDataMsg(&decoder, &req); - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); - if (pTask == NULL) { - stError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", - pMeta->vgId, req.upstreamTaskId); - return -1; - } - - int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); - if (remain > 0) { - stDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", - pTask->id.idStr, req.downstreamId, remain); - } else { - stDebug( - "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " - "completed msg", - pTask->id.idStr, req.downstreamId); - streamProcessScanHistoryFinishRsp(pTask); - } - - streamMetaReleaseTask(pMeta, pTask); - return 0; -} - -int32_t streamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - - SStreamTaskCheckReq req; - SDecoder decoder; - - tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamTaskCheckReq(&decoder, &req); - tDecoderClear(&decoder); - - int32_t taskId = req.downstreamTaskId; - - SStreamTaskCheckRsp rsp = { - .reqId = req.reqId, - .streamId = req.streamId, - .childId = req.childId, - .downstreamNodeId = req.downstreamNodeId, - .downstreamTaskId = req.downstreamTaskId, - .upstreamNodeId = req.upstreamNodeId, - .upstreamTaskId = req.upstreamTaskId, - }; - - // only the leader node handle the check request - if (pMeta->role == NODE_ROLE_FOLLOWER) { - stError("s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", - taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); - rsp.status = TASK_DOWNSTREAM_NOT_LEADER; - } else { - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); - if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); - streamMetaReleaseTask(pMeta, pTask); - - char* p = NULL; - streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", - pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - } else { - rsp.status = TASK_DOWNSTREAM_NOT_READY; - stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 - ") from task:0x%x (vgId:%d), rsp check_status %d", - req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - } - } - - return streamSendCheckRsp(pMeta, &req, &rsp, &pMsg->info, taskId); -} - -int32_t streamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { - char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t vgId = pMeta->vgId; - - int32_t code; - SStreamTaskCheckRsp rsp; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)pReq, len); - code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); - if (code < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tDecoderClear(&decoder); - stError("vgId:%d failed to parse check rsp msg, code:%s", vgId, tstrerror(terrno)); - return -1; - } - - tDecoderClear(&decoder); - stDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", - rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); - - if (!isLeader) { - stError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, - rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); - return code; - } - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); - if (pTask == NULL) { - stError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", - rsp.streamId, rsp.upstreamTaskId, vgId); - terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; - return -1; - } - - code = streamProcessCheckRsp(pTask, &rsp); - streamMetaReleaseTask(pMeta, pTask); - return code; -} - -int32_t streamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { - int32_t vgId = pMeta->vgId; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; - - SStreamCheckpointReadyMsg req = {0}; - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, len); - if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { - code = TSDB_CODE_MSG_DECODE_ERROR; - tDecoderClear(&decoder); - return code; - } - tDecoderClear(&decoder); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); - if (pTask == NULL) { - stError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); - return code; - } - - stDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId, - pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); - - streamProcessCheckpointReadyMsg(pTask); - streamMetaReleaseTask(pMeta, pTask); - return code; -} - -int32_t streamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored) { - int32_t code = 0; - int32_t vgId = pMeta->vgId; - - if (tsDisableStream) { - stInfo("vgId:%d stream disabled, not deploy stream tasks", vgId); - return code; - } - - stDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId); - - // 1.deserialize msg and build task - int32_t size = sizeof(SStreamTask); - SStreamTask* pTask = taosMemoryCalloc(1, size); - if (pTask == NULL) { - stError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size); - return TSDB_CODE_OUT_OF_MEMORY; - } - - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeStreamTask(&decoder, pTask); - tDecoderClear(&decoder); - - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pTask); - return TSDB_CODE_INVALID_MSG; - } - - // 2.save task, use the latest commit version as the initial start version of stream task. - int32_t taskId = pTask->id.taskId; - int64_t streamId = pTask->id.streamId; - bool added = false; - - streamMetaWLock(pMeta); - code = streamMetaRegisterTask(pMeta, sversion, pTask, &added); - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - streamMetaWUnLock(pMeta); - - if (code < 0) { - stError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); - tFreeStreamTask(pTask); - return code; - } - - // added into meta store, pTask cannot be reference since it may have been destroyed by other threads already now if - // it is added into the meta store - if (added) { - // only handled in the leader node - if (isLeader) { - stDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); - SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); - - if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - streamTaskHandleEvent(p->status.pSM, event); - } else if (!restored) { - stWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); - } - - if (p != NULL) { - streamMetaReleaseTask(pMeta, p); - } - } else { - stDebug("vgId:%d not leader, not launch stream task s-task:0x%x", vgId, taskId); - } - } else { - stWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store", vgId, taskId); - tFreeStreamTask(pTask); - } - - return code; -} - -int32_t streamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { - SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - - int32_t vgId = pMeta->vgId; - stDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - // drop the related fill-history task firstly - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); - stDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId); - } - streamMetaReleaseTask(pMeta, pTask); - } - - // drop the stream task now - streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); - - // commit the update - streamMetaWLock(pMeta); - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - stDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks); - - if (streamMetaCommit(pMeta) < 0) { - // persist to disk - } - streamMetaWUnLock(pMeta); - - return 0; -} - -int32_t startStreamTasks(SStreamMeta* pMeta) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = pMeta->vgId; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - stDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - taosHashClear(pMeta->startInfo.pReadyTaskSet); - taosHashClear(pMeta->startInfo.pFailedTaskSet); - pMeta->startInfo.startTs = taosGetTimestampMs(); - streamMetaWUnLock(pMeta); - - // broadcast the check downstream tasks msg - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - continue; - } - - // fill-history task can only be launched by related stream tasks. - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - if (pTask->status.downstreamReady == 1) { - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); - } - - streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); - if (ret != TSDB_CODE_SUCCESS) { - code = ret; - } - - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - return code; -} - -int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { - int32_t vgId = pMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - - stDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); - - STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; - SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - streamTaskResetStatus(*pTask); - } - - return 0; -} - -static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { - int32_t vgId = pMeta->vgId; - int32_t code = 0; - int64_t st = taosGetTimestampMs(); - - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - stDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - terrno = 0; - stInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, - pMeta->updateInfo.transId); - - while (streamMetaTaskInTimer(pMeta)) { - stDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - code = streamMetaReopen(pMeta); - if (code != TSDB_CODE_SUCCESS) { - stError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - code = terrno; - return code; - } - - streamMetaInitBackend(pMeta); - int64_t el = taosGetTimestampMs() - st; - - stInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); - - code = streamMetaLoadAllTasks(pMeta); - if (code != TSDB_CODE_SUCCESS) { - stError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); - streamMetaWUnLock(pMeta); - code = terrno; - return code; - } - - if (isLeader && !tsDisableStream) { - stInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); - resetStreamTaskStatus(pMeta); - - streamMetaWUnLock(pMeta); - startStreamTasks(pMeta); - } else { - streamMetaResetStartInfo(&pMeta->startInfo); - streamMetaWUnLock(pMeta); - stInfo("vgId:%d, follower node not start stream tasks", vgId); - } - - code = terrno; - return code; -} - -int32_t streamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { - SStreamTaskRunReq* pReq = pMsg->pCont; - - int32_t taskId = pReq->taskId; - int32_t vgId = pMeta->vgId; - - if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { - startStreamTasks(pMeta); - return 0; - } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { - restartStreamTasks(pMeta, isLeader); - return 0; - } - - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); - if (pTask != NULL) { // even in halt status, the data in inputQ must be processed - char* p = NULL; - if (streamTaskReadyToRun(pTask, &p)) { - stDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, - pTask->chkInfo.nextProcessVer); - streamExecTask(pTask); - } else { - int8_t status = streamTaskSetSchedStatusInactive(pTask); - stDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, - pTask->id.idStr, p, status); - } - - streamMetaReleaseTask(pMeta, pTask); - return 0; - } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. - // todo add one function to handle this - stError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, taskId); - return -1; - } -} From c5aba6089951ebd69af31ec09a7191c34ea5a6c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Nov 2023 09:28:43 +0800 Subject: [PATCH 30/43] fix(stream): disable the time slice usage limitation for stream task. --- source/libs/stream/src/streamExec.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index e0ee01d345..8ab8f3852e 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -281,12 +281,12 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { } // dispatch the generated results - int32_t code = handleResultBlocks(pTask, pRes, size); + /*int32_t code = */handleResultBlocks(pTask, pRes, size); int64_t el = taosGetTimestampMs() - st; // downstream task input queue is full, try in 5sec - if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED && (pTask->info.fillHistory == 1)) { return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 5000}; } @@ -294,7 +294,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { return (SScanhistoryDataInfo){TASK_SCANHISTORY_CONT, 0}; } - if (el >= STREAM_SCAN_HISTORY_TIMESLICE) { + if (el >= STREAM_SCAN_HISTORY_TIMESLICE && (pTask->info.fillHistory == 1)) { stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id, pTask->info.fillHistory, el / 1000.0); return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 100}; From 9649db9003829a45a65b8dba14e3c63544692552 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 18:49:22 +0800 Subject: [PATCH 31/43] fix(stream): fix invalid free. --- source/dnode/mnode/impl/src/mndStream.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 81cb4703e1..17c8dfe9cb 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -755,13 +755,14 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { / } static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { - SMnode * pMnode = pReq->info.node; - int32_t code = -1; - SStreamObj * pStream = NULL; - SDbObj * pDb = NULL; - SCMCreateStreamReq createStreamReq = {0}; - SStreamObj streamObj = {0}; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SStreamObj *pStream = NULL; + SStreamObj streamObj = {0}; + char *sql = NULL; + int32_t sqlLen = 0; + SCMCreateStreamReq createStreamReq = {0}; if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createStreamReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -792,8 +793,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - char* sql = NULL; - int32_t sqlLen = 0; if(createStreamReq.sql != NULL){ sqlLen = strlen(createStreamReq.sql); sql = taosMemoryMalloc(sqlLen + 1); From 6bb30822c2711315dbb480037db337e8a06a62af Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 27 Nov 2023 18:51:30 +0800 Subject: [PATCH 32/43] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndStream.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 17c8dfe9cb..feb0c3e52a 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -793,7 +793,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - if(createStreamReq.sql != NULL){ + if (createStreamReq.sql != NULL) { sqlLen = strlen(createStreamReq.sql); sql = taosMemoryMalloc(sqlLen + 1); memset(sql, 0, sqlLen + 1); @@ -880,14 +880,13 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { // reuse this function for stream if (sql != NULL && sqlLen > 0) { - auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, sql, - sqlLen); - } - else{ + auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, sql, sqlLen); + } else { char detail[1000] = {0}; sprintf(detail, "dbname:%s, stream name:%s", dbname.dbname, name.dbname); auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, detail, strlen(detail)); } + _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); @@ -896,7 +895,7 @@ _OVER: mndReleaseStream(pMnode, pStream); tFreeSCMCreateStreamReq(&createStreamReq); tFreeStreamObj(&streamObj); - if(sql != NULL){ + if (sql != NULL) { taosMemoryFreeClear(sql); } return code; From 2eec29861c6e5652d6496474e749315edbdee342 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Nov 2023 10:06:21 +0800 Subject: [PATCH 33/43] fix(util): disable avx2 version decompress timestamp. --- source/util/src/tdecompress.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index bb22772f7d..9c17000c50 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -249,6 +249,7 @@ int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelemen int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian) { +#if 0 int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; __m128i prevVal = _mm_setzero_si128(); @@ -389,11 +390,12 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen } } #endif - +#endif return 0; } -int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian) { +int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, + bool UNUSED_PARAM(bigEndian)) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; __m128i prevVal = _mm_setzero_si128(); @@ -489,11 +491,7 @@ int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelem if (nbytes == 0) { deltaOfDelta = 0; } else { - // if (is_bigendian()) { - // memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); - // } else { memcpy(&dd, input + ipos, nbytes); - // } deltaOfDelta = ZIGZAG_DECODE(int64_t, dd); } From dcb5dfb7fb0987ca7cb503d6a9aea9da3af6d937 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 28 Nov 2023 10:24:03 +0800 Subject: [PATCH 34/43] test:update tmq cases and add performance testcase --- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmqCommon.py | 6 +- ...odeSplit-stb-select-duplicatedata-false.py | 218 ++++++++++++++++++ .../tmqVnodeSplit-stb-select-duplicatedata.py | 2 - tests/system-test/7-tmq/tmq_per.py | 196 ++++++++++++++++ 5 files changed, 418 insertions(+), 5 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py create mode 100644 tests/system-test/7-tmq/tmq_per.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ca1cc704ae..e37f7882c9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -198,6 +198,7 @@ #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 059744caf0..66c88cf600 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -75,7 +75,7 @@ class TMQCom: if tdSql.getRows() == expectRows: break else: - time.sleep(5) + time.sleep(0.5) for i in range(expectRows): tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) @@ -156,7 +156,7 @@ class TMQCom: tdLog.info("row: %d"%(actRows)) if (actRows >= rows): loopFlag = 0 - time.sleep(0.02) + time.sleep(0.5) return def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=1): @@ -167,7 +167,7 @@ class TMQCom: tdLog.info("row: %d"%(actRows)) if (actRows >= rows): loopFlag = 0 - time.sleep(0.02) + time.sleep(0.5) return def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py new file mode 100644 index 0000000000..8276ae638b --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -0,0 +1,218 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 120, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 4f3f46725a..0d247b2848 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -207,8 +207,6 @@ class TDTestCase: def run(self): self.prepareTestEnv() self.tmqCase1(True) - self.prepareTestEnv() - self.tmqCase1(False) def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmq_per.py b/tests/system-test/7-tmq/tmq_per.py new file mode 100644 index 0000000000..f3701dacab --- /dev/null +++ b/tests/system-test/7-tmq/tmq_per.py @@ -0,0 +1,196 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10000 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10000, + 'rowsPerTbl': 10000, + 'batchNum': 10000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + print + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb ") + queryString = "stable %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + expectRows = 1 + tdLog.info("expectRows:%d"%expectRows) + resultList = tmqCom.selectConsumeResult(expectRows) + # for i in range(len(topicNameList)): + # tdSql.query("drop topic %s"%topicNameList[i]) + + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=1,db_replica=self.replicaVar,db_name="dbt",count_number=240) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From f4cf898af73c7728b5dc2ea7e7124f6e4854e0f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Nov 2023 13:59:13 +0800 Subject: [PATCH 35/43] fix(util): fix syntax error. --- source/util/src/tdecompress.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 9c17000c50..f212bf5231 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -398,11 +398,12 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem bool UNUSED_PARAM(bigEndian)) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; - __m128i prevVal = _mm_setzero_si128(); - __m128i prevDelta = _mm_setzero_si128(); #if __AVX512VL__ + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); + int32_t numOfBatch = nelements >> 1; int32_t remainder = nelements & 0x01; __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; From a73992e12833b4dec874dcab47d28ed314fab4cd Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 28 Nov 2023 14:11:19 +0800 Subject: [PATCH 36/43] stream event reload state --- source/libs/executor/src/streameventwindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 9b987ff1a4..8029b9b156 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -603,7 +603,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { pSeKeyBuf[i].groupId, i); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); setEventWindowFlag(pAggSup, &curInfo); - if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { + if (!curInfo.pWinFlag->startFlag || !curInfo.pWinFlag->endFlag) { continue; } From 2ac691aa29fc7649055803581486052c6b4c1aae Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 28 Nov 2023 15:18:06 +0800 Subject: [PATCH 37/43] test:test slave50,59 and 63 in ci --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 8ddee6dbbd..1ab6a4a8cd 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -355,7 +355,7 @@ pipeline { } parallel { stage('check docs') { - agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 || Mac_catalina "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 || Mac_catalina "} steps { check_docs() } @@ -401,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() From b54f5b47ad4f2511cb932ebbce44bce50238d215 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 28 Nov 2023 17:58:54 +0800 Subject: [PATCH 38/43] fix: reserve enough buffer for user --- source/dnode/mnode/impl/src/mndUser.c | 31 +++++++++++++------- tests/system-test/0-others/user_privilege.py | 19 +++++++++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 66abfd6bc1..0e3b544508 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -708,9 +708,8 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews); int32_t numOfTopics = taosHashGetSize(pUser->topics); int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); - int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + - (numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN + - ipWhiteReserve; + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN + + numOfTopics * TSDB_TOPIC_FNAME_LEN + ipWhiteReserve; char *stb = taosHashIterate(pUser->readTbs, NULL); while (stb != NULL) { @@ -720,7 +719,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->readTbs, stb); @@ -734,7 +733,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->writeTbs, stb); @@ -748,7 +747,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->alterTbs, stb); @@ -762,7 +761,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->readViews, stb); @@ -776,7 +775,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->writeViews, stb); @@ -790,11 +789,21 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size += keyLen; size_t valueLen = 0; - valueLen = strlen(stb); + valueLen = strlen(stb) + 1; size += sizeof(int32_t); size += valueLen; stb = taosHashIterate(pUser->alterViews, stb); - } + } + + int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); + while (useDb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(useDb, &keyLen); + size += sizeof(int32_t); + size += keyLen; + size += sizeof(int32_t); + useDb = taosHashIterate(pUser->useDbs, useDb); + } SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -925,7 +934,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { stb = taosHashIterate(pUser->alterViews, stb); } - int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); + useDb = taosHashIterate(pUser->useDbs, NULL); while (useDb != NULL) { size_t keyLen = 0; void *key = taosHashGetKey(useDb, &keyLen); diff --git a/tests/system-test/0-others/user_privilege.py b/tests/system-test/0-others/user_privilege.py index d1b93f6942..a731e85ddb 100644 --- a/tests/system-test/0-others/user_privilege.py +++ b/tests/system-test/0-others/user_privilege.py @@ -27,6 +27,7 @@ class TDTestCase: tdSql.init(conn.cursor()) self.setsql = TDSetSql() self.stbname = 'stb' + self.user_name = 'test' self.binary_length = 20 # the length of binary for column_dict self.nchar_length = 20 # the length of nchar for column_dict self.dbnames = ['db1', 'db2'] @@ -54,12 +55,12 @@ class TDTestCase: ] self.tbnum = 4 + self.stbnum_grant = 200 def create_user(self): - user_name = 'test' - tdSql.execute(f'create user {user_name} pass "test"') - tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {user_name}') - tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {user_name}') + tdSql.execute(f'create user {self.user_name} pass "test"') + tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {self.user_name}') + tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {self.user_name}') def prepare_data(self): for db in self.dbnames: @@ -70,6 +71,8 @@ class TDTestCase: tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') for j in self.values_list: tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for i in range(self.stbnum_grant): + tdSql.execute(f'create table {self.stbname}_grant_{i} (ts timestamp, c0 int) tags(t0 int)') def user_read_privilege_check(self, dbname): testconn = taos.connect(user='test', password='test') @@ -128,12 +131,20 @@ class TDTestCase: tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured") pass + def user_privilege_grant_check(self): + for db in self.dbnames: + tdSql.execute(f"use {db}") + for i in range(self.stbnum_grant): + tdSql.execute(f'grant read on {db}.{self.stbname}_grant_{i} to {self.user_name}') + tdSql.execute(f'grant write on {db}.{self.stbname}_grant_{i} to {self.user_name}') + def run(self): self.prepare_data() self.create_user() self.user_read_privilege_check(self.dbnames[0]) self.user_write_privilege_check(self.dbnames[1]) self.user_privilege_error_check() + self.user_privilege_grant_check() def stop(self): tdSql.close() From a5c8eaacaac548ef67d3e395164c21d556fc1a7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Nov 2023 23:52:19 +0800 Subject: [PATCH 39/43] fix(tsdb):clear the tsdb stt file reader flag when suspending the tsdbReader --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 67 +++++++++------------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 1 + 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 751df706ab..5d018af6c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4148,6 +4148,20 @@ void tsdbReaderClose2(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } +static void clearMemIterInfo(STableBlockScanInfo* pInfo) { + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } +} + int32_t tsdbReaderSuspend2(STsdbReader* pReader) { // save reader's base state & reset top state to be reconstructed from base state int32_t code = 0; @@ -4168,28 +4182,20 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { tsdbDataFileReaderClose(&pReader->pFileReader); SCostSummary* pCost = &pReader->cost; + pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; - - pInfo->iterInit = false; - pInfo->iter.hasVal = false; - pInfo->iiter.hasVal = false; - - if (pInfo->iter.iter != NULL) { - pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); - } - - if (pInfo->iiter.iter != NULL) { - pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); - } - + clearMemIterInfo(pInfo); + pInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + pInfo->pFileDelData = taosArrayDestroy(pInfo->pFileDelData); } } else { @@ -4199,45 +4205,24 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; - - pInfo->iterInit = false; - pInfo->iter.hasVal = false; - pInfo->iiter.hasVal = false; - - if (pInfo->iter.iter != NULL) { - pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); - } - - if (pInfo->iiter.iter != NULL) { - pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); - } - + clearMemIterInfo(pInfo); + pInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); } - pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; + pBlockScanInfo = (pStatus->pTableIter == NULL) ? NULL : *pStatus->pTableIter; if (pBlockScanInfo) { // save lastKey to restore memory iterator STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; pBlockScanInfo->lastProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; - // reset current current table's data block scan info, - pBlockScanInfo->iterInit = false; - - pBlockScanInfo->iter.hasVal = false; - pBlockScanInfo->iiter.hasVal = false; - if (pBlockScanInfo->iter.iter != NULL) { - pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); - } - - if (pBlockScanInfo->iiter.iter != NULL) { - pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); - } + clearMemIterInfo(pBlockScanInfo); + pBlockScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; + pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); pBlockScanInfo->pBlockIdxList = taosArrayDestroy(pBlockScanInfo->pBlockIdxList); // TODO: keep skyline for reuse - pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 305399e0af..24c526a906 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -210,6 +210,7 @@ void clearBlockScanInfo(STableBlockScanInfo* p) { p->iterInit = false; p->iter.hasVal = false; p->iiter.hasVal = false; + p->sttKeyInfo.status = STT_FILE_READER_UNINIT; if (p->iter.iter != NULL) { p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); From 1e02f823a42e023c7d439fafc2e1db72030ec83f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 29 Nov 2023 14:59:22 +0800 Subject: [PATCH 40/43] compact history window --- .../libs/executor/src/streameventwindowoperator.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 8029b9b156..9f1610e08d 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -603,13 +603,20 @@ void streamEventReloadState(SOperatorInfo* pOperator) { pSeKeyBuf[i].groupId, i); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); setEventWindowFlag(pAggSup, &curInfo); - if (!curInfo.pWinFlag->startFlag || !curInfo.pWinFlag->endFlag) { + if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { continue; } compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + } + + if (!curInfo.pWinFlag->endFlag) { + continue; + } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { saveResult(curInfo.winInfo, pInfo->pSeUpdated); @@ -621,10 +628,6 @@ void streamEventReloadState(SOperatorInfo* pOperator) { getSessionHashKey(&curInfo.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); } - - if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); - } } taosMemoryFree(pBuf); From 121197b8977da5136cd9124e1e5ed995c3c540ba Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:02:18 +0800 Subject: [PATCH 41/43] Update set_core.sh --- packaging/tools/set_core.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/tools/set_core.sh b/packaging/tools/set_core.sh index db95aeb343..084c4465f9 100755 --- a/packaging/tools/set_core.sh +++ b/packaging/tools/set_core.sh @@ -38,3 +38,4 @@ source /etc/profile ${csudo}mkdir -p ${corePath} ||: ${csudo}sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: ${csudo}echo "${corePath}/core-%e-%p" | ${csudo}tee /proc/sys/kernel/core_pattern ||: +${csudo}echo "kernel.core_pattern = ${corePath}/core_%e-%p" >> /etc/sysctl.conf ||: From 66f25712fd4f1e9e710e296f022509461e21ebb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Nov 2023 18:07:15 +0800 Subject: [PATCH 42/43] fix(tsdb): fix bugs in suspend/resume. adjust code to conduct the test more easily. td-27579 --- source/client/test/clientTests.cpp | 102 ++++++++------ source/dnode/vnode/src/inc/vnodeInt.h | 4 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 151 ++++++++------------- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 14 +- 4 files changed, 127 insertions(+), 144 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index addf0aa629..e78783cf3c 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -126,9 +126,9 @@ void queryCallback(void* param, void* res, int32_t code) { taos_fetch_raw_block_a(res, fetchCallback, param); } -void createNewTable(TAOS* pConn, int32_t index) { +void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t startTs, const char* pVarchar) { char str[1024] = {0}; - sprintf(str, "create table tu%d using st2 tags(%d)", index, index); + sprintf(str, "create table if not exists tu%d using st2 tags(%d)", index, index); TAOS_RES* pRes = taos_query(pConn, str); if (taos_errno(pRes) != 0) { @@ -136,22 +136,43 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 10000; i += 20) { - char sql[1024] = {0}; - sprintf(sql, - "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, - i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, - i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } + if (startTs == 0) { + for (int32_t i = 0; i < numOfRows; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, + i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, + i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } - taos_free_result(p); + taos_free_result(p); + } + } else { + for (int32_t i = 0; i < numOfRows; i += 20) { + char sql[1024*50] = {0}; + sprintf(sql, + "insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, " + "%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, " + "'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')", + index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4, + pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9, + pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14, + i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18, + pVarchar, startTs + 19, i + 19, pVarchar); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } } } @@ -808,14 +829,7 @@ TEST(clientCase, projection_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int, f varchar(4096)) tags(a int)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } @@ -828,28 +842,32 @@ TEST(clientCase, projection_query_tables) { taos_free_result(pRes); int64_t start = 1685959190000; + const char* pstr = + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefgh" + "ijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnop" + "qrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwx" + "yzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdef" + "ghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz!@#$%^&&*&^^%$#@!qQWERTYUIOPASDFGHJKL:" + "QWERTYUIOP{}"; - int32_t code = -1; - for(int32_t i = 0; i < 1000000; ++i) { - char t[512] = {0}; + for(int32_t i = 0; i < 10000; ++i) { + char str[1024] = {0}; + sprintf(str, "create table if not exists tu%d using st2 tags(%d)", i, i); - sprintf(t, "insert into t1 values(now, %d)", i); - while(1) { - void* p = taos_query(pConn, t); - code = taos_errno(p); - taos_free_result(p); - if (code != 0) { - printf("insert data error, retry\n"); - } else { - break; - } + TAOS_RES* px = taos_query(pConn, str); + if (taos_errno(px) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(px); + } + + for(int32_t j = 0; j < 5000; ++j) { + start += 20; + for (int32_t i = 0; i < 10000; ++i) { + createNewTable(pConn, i, 20, start, pstr); } } - for (int32_t i = 0; i < 1; ++i) { - printf("create table :%d\n", i); - createNewTable(pConn, i); - } // // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 16379db053..f3f84896ad 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -93,7 +93,11 @@ typedef struct SQueryNode SQueryNode; #define VNODE_RSMA2_DIR "rsma2" #define VNODE_TQ_STREAM "stream" +#if SUSPEND_RESUME_TEST // only for test purpose +#define VNODE_BUFPOOL_SEGMENTS 1 +#else #define VNODE_BUFPOOL_SEGMENTS 3 +#endif #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5d018af6c1..4138d91f5d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -48,7 +48,7 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScan static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); -static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost); +static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); @@ -58,6 +58,7 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbRea static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); +static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -168,7 +169,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo return TSDB_CODE_SUCCESS; } - SCostSummary* pCost = &pReader->cost; + SReadCostSummary* pCost = &pReader->cost; pIter->pLastBlockReader->uid = 0; tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); @@ -291,11 +292,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) } static int32_t tsdbInitReaderLock(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - - code = taosThreadMutexInit(&pReader->readerMutex, NULL); - + int32_t code = taosThreadMutexInit(&pReader->readerMutex, NULL); qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; @@ -324,22 +321,14 @@ static int32_t tsdbAcquireReader(STsdbReader* pReader) { } static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - - code = taosThreadMutexTryLock(&pReader->readerMutex); - + int32_t code = taosThreadMutexTryLock(&pReader->readerMutex); qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; } static int32_t tsdbReleaseReader(STsdbReader* pReader) { - int32_t code = -1; - qTrace("tsdb/read: %p, pre-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); - - code = taosThreadMutexUnlock(&pReader->readerMutex); - + int32_t code = taosThreadMutexUnlock(&pReader->readerMutex); qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); return code; @@ -432,6 +421,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void } tsdbInitReaderLock(pReader); + tsem_init(&pReader->resumeAfterSuspend, 0, 0); *ppReader = pReader; return code; @@ -1015,8 +1005,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pRecord->numRow) { int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - if (outOfTimeWindow(ts, - &pReader->info.window)) { // the remain data has out of query time window, ignore current block + if (outOfTimeWindow(ts, &pReader->info.window)) { + // the remain data has out of query time window, ignore current block setBlockAllDumped(pDumpInfo, ts, pReader->info.order); } } else { @@ -1123,16 +1113,12 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo } int32_t step = asc ? 1 : -1; - // *nextIndex = pBlockInfo->tbBlockIdx + step; - // *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); memcpy(pRecord, &p->record, sizeof(SBrinRecord)); *nextIndex = pBlockInfo->tbBlockIdx + step; - - // tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); return true; } @@ -1376,23 +1362,19 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; - int64_t st = taosGetTimestampUs(); + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); - blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); - pBlock->info.id.uid = pBlockScanInfo->uid; + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pBlockScanInfo); - setComposedBlockFlag(pReader, true); - - double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", - pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, + pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, pReader->idStr); - pReader->cost.buildmemBlock += elapsedTime; + pReader->cost.buildmemBlock += el; return code; } @@ -2293,13 +2275,12 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock return code; } -static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { +void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; pResBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); - setComposedBlockFlag(pReader, true); pReader->cost.composedBlocks += 1; @@ -2356,7 +2337,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pBlockScanInfo = *pReader->status.pTableIter; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { - // setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->info.order); return code; } } @@ -2436,7 +2416,7 @@ int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) { return ASCENDING_TRAVERSE(order) ? 0 : taosArrayGetSize(pDelSkyline) - 1; } -int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost) { +int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost) { int32_t code = 0; int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData); if (newDelDataInFile == 0 && @@ -2935,6 +2915,8 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; + tsdbDebug("seq load data blocks from cache, %s", pReader->idStr); + while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); @@ -3043,6 +3025,8 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SDataBlockIter* pBlockIter = &pReader->status.blockIter; + tsdbDebug("seq load data blocks from stt files %s", pReader->idStr); + while (1) { terrno = 0; @@ -3774,7 +3758,6 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e int32_t code = TSDB_CODE_SUCCESS; do { - // SRow* pTSRow = NULL; TSDBROW row = {.type = -1}; bool freeTSRow = false; tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); @@ -3792,13 +3775,17 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e if (code) { return code; } + + pBlockScanInfo->lastProcKey = row.pTSRow->ts; } else { code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); if (code) { break; } + pBlockScanInfo->lastProcKey = row.pBlockData->aTSKEY[row.iRow]; } + // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { break; @@ -4107,7 +4094,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { tsdbDataFileReaderClose(&pReader->pFileReader); } - SCostSummary* pCost = &pReader->cost; + SReadCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; @@ -4122,6 +4109,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, true); pReader->pReadSnap = NULL; + tsem_destroy(&pReader->resumeAfterSuspend); tsdbReleaseReader(pReader); tsdbUninitReaderLock(pReader); @@ -4148,26 +4136,14 @@ void tsdbReaderClose2(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } -static void clearMemIterInfo(STableBlockScanInfo* pInfo) { - pInfo->iterInit = false; - pInfo->iter.hasVal = false; - pInfo->iiter.hasVal = false; - - if (pInfo->iter.iter != NULL) { - pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); - } - - if (pInfo->iiter.iter != NULL) { - pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); - } -} - int32_t tsdbReaderSuspend2(STsdbReader* pReader) { // save reader's base state & reset top state to be reconstructed from base state int32_t code = 0; SReaderStatus* pStatus = &pReader->status; STableBlockScanInfo* pBlockScanInfo = NULL; + pReader->status.suspendInvoked = true; // record the suspend status + if (pStatus->loadFromFile) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); if (pBlockInfo != NULL) { @@ -4181,55 +4157,34 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { tsdbDataFileReaderClose(&pReader->pFileReader); - SCostSummary* pCost = &pReader->cost; - + SReadCostSummary* pCost = &pReader->cost; pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - - // resetDataBlockScanInfo excluding lastKey - STableBlockScanInfo** p = NULL; - - int32_t iter = 0; - while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { - STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; - clearMemIterInfo(pInfo); - pInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - - pInfo->pFileDelData = taosArrayDestroy(pInfo->pFileDelData); - } - } else { - // resetDataBlockScanInfo excluding lastKey - STableBlockScanInfo** p = NULL; - int32_t iter = 0; - - while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { - STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; - clearMemIterInfo(pInfo); - pInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - } - - pBlockScanInfo = (pStatus->pTableIter == NULL) ? NULL : *pStatus->pTableIter; - if (pBlockScanInfo) { - // save lastKey to restore memory iterator - STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; - pBlockScanInfo->lastProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; - - clearMemIterInfo(pBlockScanInfo); - pBlockScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); - - pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); - pBlockScanInfo->pBlockIdxList = taosArrayDestroy(pBlockScanInfo->pBlockIdxList); - // TODO: keep skyline for reuse - } } + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + + int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1:-1; + + int32_t iter = 0; + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + clearBlockScanInfo(pInfo); + pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step; + } + + pStatus->uidList.currentIndex = 0; + initReaderStatus(pStatus); + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; pReader->flag = READER_STATUS_SUSPEND; +#if SUSPEND_RESUME_TEST + tsem_post(&pReader->resumeAfterSuspend); +#endif + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, pReader->idStr); return code; @@ -4384,6 +4339,16 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { SReaderStatus* pStatus = &pReader->status; + // NOTE: the following codes is used to perform test for suspend/resume for tsdbReader when it blocks the commit + // the data should be ingested in round-robin and all the child tables should be createted before ingesting data + // the version range of query will be used to identify the correctness of suspend/resume functions. + // this function will blocked before loading the SECOND block from vnode-buffer, and restart itself from sst-files +#if SUSPEND_RESUME_TEST + if (!pReader->status.suspendInvoked && !pReader->status.loadFromFile) { + tsem_wait(&pReader->resumeAfterSuspend); + } +#endif + code = tsdbAcquireReader(pReader); qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 60e6e6960a..709e311ff0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -96,7 +96,7 @@ typedef struct SResultBlockInfo { int64_t capacity; } SResultBlockInfo; -typedef struct SCostSummary { +typedef struct SReadCostSummary { int64_t numOfBlocks; double blockLoadTime; double buildmemBlock; @@ -110,7 +110,7 @@ typedef struct SCostSummary { double createScanInfoList; double createSkylineIterTime; double initLastBlockReader; -} SCostSummary; +} SReadCostSummary; typedef struct STableUidList { uint64_t* tableUidList; // access table uid list in uid ascending order list @@ -122,12 +122,6 @@ typedef struct { int32_t numOfSttFiles; } SBlockNumber; -typedef struct SBlockIndex { - int32_t ordinalIndex; - int64_t inFileOffset; - STimeWindow window; // todo replace it with overlap flag. -} SBlockIndex; - typedef struct SBlockOrderWrapper { int64_t uid; int64_t offset; @@ -192,6 +186,7 @@ typedef struct SFileBlockDumpInfo { } SFileBlockDumpInfo; typedef struct SReaderStatus { + bool suspendInvoked; bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not SSHashObj* pTableMap; // SHash @@ -220,7 +215,8 @@ struct STsdbReader { int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; - SCostSummary cost; + tsem_t resumeAfterSuspend; + SReadCostSummary cost; SHashObj** pIgnoreTables; SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFileReader* pFileReader; // the file reader From 28e17a61b932000c32c05d1ffe5c8a444e926cb3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Nov 2023 19:28:53 +0800 Subject: [PATCH 43/43] fix(tsdb): fix invalid read. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 4138d91f5d..cb899f9ee8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -3766,6 +3766,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } if (row.type == TSDBROW_ROW_FMT) { + int64_t ts = row.pTSRow->ts;; code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); if (freeTSRow) { @@ -3776,7 +3777,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e return code; } - pBlockScanInfo->lastProcKey = row.pTSRow->ts; + pBlockScanInfo->lastProcKey = ts; } else { code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); if (code) {