From 3de7530a54e87eb7bc51309ba03e041980425316 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 27 Jan 2022 11:50:03 +0800 Subject: [PATCH 01/42] add queue test --- source/util/test/CMakeLists.txt | 8 +++++++- source/util/test/queueTest.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 source/util/test/queueTest.cpp diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 383a00232a..ee0ade03b8 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -45,4 +45,10 @@ target_link_libraries(freelistTest os util gtest gtest_main) add_executable(encodeTest "encodeTest.cpp") target_link_libraries(encodeTest os util gtest gtest_main) - +# queueTest +add_executable(queue_test "queueTest.cpp") +target_link_libraries(queue_test os util gtest_main) +add_test( + NAME queue_test + COMMAND queue_test +) \ No newline at end of file diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp new file mode 100644 index 0000000000..310ae4350e --- /dev/null +++ b/source/util/test/queueTest.cpp @@ -0,0 +1,29 @@ +/** + * @file queue.cpp + * @author slguan (slguan@taosdata.com) + * @brief UTIL module queue tests + * @version 1.0 + * @date 2022-01-27 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "os.h" +#include "tqueue.h" + +class UtilTestQueue : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} +}; + +TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { + EXPECT_EQ(0, 0); +} \ No newline at end of file From 45c4099e992ee30de01250a9ac91c332138429e9 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 27 Jan 2022 01:32:40 -0500 Subject: [PATCH 02/42] TD-13197 SELECT statement syntax definition --- include/common/ttokendef.h | 76 +- include/nodes/nodes.h | 51 +- source/libs/parser/inc/astCreateContext.h | 10 +- source/libs/parser/inc/astCreateFuncs.h | 33 +- source/libs/parser/inc/new_sql.y | 345 ++++-- source/libs/parser/src/astCreateContext.c | 48 +- source/libs/parser/src/astCreateFuncs.c | 222 +++- source/libs/parser/src/new_sql.c | 1304 +++++++++++++-------- source/libs/parser/src/parserImpl.c | 7 +- source/libs/parser/test/newParserTest.cpp | 26 +- source/nodes/src/nodesUtilFuncs.c | 45 +- 11 files changed, 1405 insertions(+), 762 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index cbc8c0e87c..f8aa7df02a 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -207,36 +207,50 @@ #define TK_INTO 189 #define TK_VALUES 190 -#define NEW_TK_UNION 1 -#define NEW_TK_ALL 2 -#define NEW_TK_MINUS 3 -#define NEW_TK_EXCEPT 4 -#define NEW_TK_INTERSECT 5 -#define NEW_TK_NK_PLUS 6 -#define NEW_TK_NK_MINUS 7 -#define NEW_TK_NK_STAR 8 -#define NEW_TK_NK_SLASH 9 -#define NEW_TK_SHOW 10 -#define NEW_TK_DATABASES 11 -#define NEW_TK_NK_ID 12 -#define NEW_TK_NK_LP 13 -#define NEW_TK_NK_RP 14 -#define NEW_TK_NK_COMMA 15 -#define NEW_TK_NK_LITERAL 16 -#define NEW_TK_NK_DOT 17 -#define NEW_TK_SELECT 18 -#define NEW_TK_DISTINCT 19 -#define NEW_TK_AS 20 -#define NEW_TK_FROM 21 -#define NEW_TK_WITH 22 -#define NEW_TK_RECURSIVE 23 -#define NEW_TK_ORDER 24 -#define NEW_TK_BY 25 -#define NEW_TK_ASC 26 -#define NEW_TK_DESC 27 -#define NEW_TK_NULLS 28 -#define NEW_TK_FIRST 29 -#define NEW_TK_LAST 30 +#define NEW_TK_OR 1 +#define NEW_TK_AND 2 +#define NEW_TK_NOT 3 +#define NEW_TK_UNION 4 +#define NEW_TK_ALL 5 +#define NEW_TK_MINUS 6 +#define NEW_TK_EXCEPT 7 +#define NEW_TK_INTERSECT 8 +#define NEW_TK_NK_PLUS 9 +#define NEW_TK_NK_MINUS 10 +#define NEW_TK_NK_STAR 11 +#define NEW_TK_NK_SLASH 12 +#define NEW_TK_SHOW 13 +#define NEW_TK_DATABASES 14 +#define NEW_TK_NK_INTEGER 15 +#define NEW_TK_NK_FLOAT 16 +#define NEW_TK_NK_STRING 17 +#define NEW_TK_NK_BOOL 18 +#define NEW_TK_NK_NOW 19 +#define NEW_TK_NK_ID 20 +#define NEW_TK_NK_QUESTION 21 +#define NEW_TK_NK_LP 22 +#define NEW_TK_NK_RP 23 +#define NEW_TK_NK_DOT 24 +#define NEW_TK_FROM 25 +#define NEW_TK_NK_COMMA 26 +#define NEW_TK_AS 27 +#define NEW_TK_JOIN 28 +#define NEW_TK_ON 29 +#define NEW_TK_INNER 30 +#define NEW_TK_SELECT 31 +#define NEW_TK_DISTINCT 32 +#define NEW_TK_ORDER 33 +#define NEW_TK_BY 34 +#define NEW_TK_SLIMIT 35 +#define NEW_TK_SOFFSET 36 +#define NEW_TK_LIMIT 37 +#define NEW_TK_OFFSET 38 +#define NEW_TK_NK_LR 39 +#define NEW_TK_ASC 40 +#define NEW_TK_DESC 41 +#define NEW_TK_NULLS 42 +#define NEW_TK_FIRST 43 +#define NEW_TK_LAST 44 #define TK_SPACE 300 #define TK_COMMENT 301 @@ -247,6 +261,8 @@ #define TK_FILE 306 #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query +#define TK_NIL 65535 + #endif diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index e297059bc8..5fb4c3476d 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -22,6 +22,19 @@ extern "C" { #include "tdef.h" +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) + +#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) + +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) + +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) + typedef enum ENodeType { QUERY_NODE_COLUMN = 1, QUERY_NODE_VALUE, @@ -52,9 +65,6 @@ typedef struct SNode { ENodeType type; } SNode; -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) -#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) - typedef struct SListCell { SNode* pNode; struct SListCell* pNext; @@ -62,19 +72,10 @@ typedef struct SListCell { typedef struct SNodeList { int16_t length; - SListCell* pHeader; + SListCell* pHead; + SListCell* pTail; } SNodeList; -#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) - -#define FOREACH(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHeader : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) - -#define FORBOTH(node1, list1, node2, list2) \ - for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHeader : NULL), *cell2 = (NULL != (list2) ? (list2)->pHeader : NULL); \ - (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ - cell1 = cell1->pNext, cell2 = cell2->pNext) - typedef struct SDataType { uint8_t type; uint8_t precision; @@ -252,13 +253,14 @@ typedef struct SSelectStmt { bool isStar; SNodeList* pProjectionList; // SNode SNode* pFromTable; - SNode* pWhereCond; + SNode* pWhere; SNodeList* pPartitionByList; // SNode - SNode* pWindowClause; + SNode* pWindow; SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; SNodeList* pOrderByList; // SOrderByExprNode - SLimitNode limit; - SLimitNode slimit; + SLimitNode* pLimit; + SLimitNode* pSlimit; } SSelectStmt; typedef enum ESetOperatorType { @@ -272,10 +274,17 @@ typedef struct SSetOperator { SNode* pRight; } SSetOperator; +SNode* nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNode* pNode); + +SNodeList* nodesMakeList(); +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); +void nodesDestroyList(SNodeList* pList); + typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); -bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext); +bool nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); @@ -289,10 +298,6 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -SNode* nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNode* pNode); -void nodesDestroyNodeList(SNodeList* pList); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/astCreateContext.h b/source/libs/parser/inc/astCreateContext.h index dcdc6a045b..107802bbc4 100644 --- a/source/libs/parser/inc/astCreateContext.h +++ b/source/libs/parser/inc/astCreateContext.h @@ -23,17 +23,23 @@ extern "C" { #include "nodes.h" #include "parser.h" +typedef enum EResourceType { + AST_CXT_RESOURCE_NODE = 1, + AST_CXT_RESOURCE_NODE_LIST +} EResourceType; + typedef struct SAstCreateContext { SParseContext* pQueryCxt; bool notSupport; bool valid; SNode* pRootNode; + SHashObj* pResourceHash; } SAstCreateContext; -int32_t createAstCreateContext(const SParseContext* pQueryCxt, SAstCreateContext* pCxt); +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt); int32_t destroyAstCreateContext(SAstCreateContext* pCxt); -void* acquireRaii(SAstCreateContext* pCxt, void* p); +void* acquireRaii(SAstCreateContext* pCxt, EResourceType type, void* p); void* releaseRaii(SAstCreateContext* pCxt, void* p); #ifdef __cplusplus diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index c4c5dcba6c..515df7f7fe 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -25,20 +25,35 @@ extern "C" { #endif -bool checkTableName(const SToken* pTableName); -SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); -SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); -SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); -SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +extern SToken nil_token; + SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); +SNode* createLimitNode(SAstCreateContext* pCxt, SNode* pLimit, SNode* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName); + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow); +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList); +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving); +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); + SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type); -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); + #ifdef __cplusplus } diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index 9237f63f35..0fc316dd29 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -6,7 +6,7 @@ %token_prefix NEW_TK_ %token_type { SToken } %default_type { SNode* } -%default_destructor { nodesDestroyNode($$); } +%default_destructor { PARSER_DESTRUCTOR_TRACE; nodesDestroyNode($$); } %extra_argument { SAstCreateContext* pCxt } @@ -22,7 +22,8 @@ #include "ttokendef.h" #include "astCreateFuncs.h" -#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) } %syntax_error { @@ -45,9 +46,9 @@ %parse_accept { printf("parsing complete!\n" );} -//%left OR. -//%left AND. -//%right NOT. +%left OR. +%left AND. +%right NOT. %left UNION ALL MINUS EXCEPT INTERSECT. //%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. //%left GT GE LT LE. @@ -58,27 +59,226 @@ //%left CONCAT. //%right UMINUS UPLUS BITNOT. -cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } -cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } +/*5.4*********************************************** literal *********************************************************/ +unsigned_integer(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +unsigned_approximate_numeric(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } + +signed_integer(A) ::= unsigned_integer(B). { A = B; } +signed_integer(A) ::= NK_PLUS unsigned_integer(B). { A = B; } +signed_integer(A) ::= NK_MINUS unsigned_integer(B). { A = addMinusSign(pCxt, B);} + +unsigned_literal ::= unsigned_numeric_literal. +unsigned_literal ::= general_literal. + +unsigned_numeric_literal ::= unsigned_integer. +unsigned_numeric_literal ::= unsigned_approximate_numeric. + +general_literal ::= NK_STRING. +general_literal ::= NK_BOOL. +general_literal ::= NK_NOW. + +/*5.4*********************************************** names and identifiers *********************************************************/ +%type db_name { SToken } +%destructor db_name { PARSER_DESTRUCTOR_TRACE; } +db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type table_name { SToken } +%destructor table_name { PARSER_DESTRUCTOR_TRACE; } +table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type column_name { SToken } +%destructor column_name { PARSER_DESTRUCTOR_TRACE; } +column_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type table_alias { SToken } +%destructor table_alias { PARSER_DESTRUCTOR_TRACE; } +table_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +/*6.4*********************************************** value_specification *********************************************************/ +unsigned_value_specification ::= unsigned_literal. +unsigned_value_specification ::= NK_QUESTION. + +/*6.35 todo *********************************************** value_expression_primary *********************************************************/ +value_expression_primary ::= NK_LP value_expression NK_RP. +value_expression_primary ::= nonparenthesized_value_expression_primary. + +nonparenthesized_value_expression_primary ::= unsigned_value_specification. +nonparenthesized_value_expression_primary ::= column_reference. +//nonparenthesized_value_expression_primary ::= agg_function. +nonparenthesized_value_expression_primary ::= subquery. +//nonparenthesized_value_expression_primary ::= case_expression. // todo +//nonparenthesized_value_expression_primary ::= cast_specification. // todo + +column_reference(A) ::= column_name(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } +column_reference(A) ::= table_name(B) NK_DOT column_name(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } + +/*6.35*********************************************** boolean_value_expression *********************************************************/ +boolean_value_expression(A) ::= boolean_primary(B). { PARSER_TRACE; A = B; } +boolean_value_expression(A) ::= NOT boolean_primary(B). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); } +boolean_value_expression(A) ::= + boolean_value_expression(B) OR boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, B, C); } +boolean_value_expression(A) ::= + boolean_value_expression(B) AND boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, B, C); } + +boolean_primary(A) ::= predicate(B). { PARSER_TRACE; A = B; } +boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP. { PARSER_TRACE; A = B; } + +/*7.5*********************************************** from_clause *********************************************************/ +from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } + +table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } +table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } + +/*7.6*********************************************** table_reference *****************************************************/ +table_reference(A) ::= table_primary(B). { PARSER_TRACE; A = B; } +table_reference(A) ::= joined_table(B). { PARSER_TRACE; A = B; } + +table_primary(A) ::= table_name(B) correlation_or_recognition_opt(C). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= + db_name(B) NK_DOT table_name(C) correlation_or_recognition_opt(D). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) correlation_or_recognition_opt(C). { PARSER_TRACE; A = createTempTableNode(pCxt, B, &C); } +table_primary ::= parenthesized_joined_table. + +%type correlation_or_recognition_opt { SToken } +%destructor correlation_or_recognition_opt { PARSER_DESTRUCTOR_TRACE; } +correlation_or_recognition_opt(A) ::= . { PARSER_TRACE; A = nil_token; } +correlation_or_recognition_opt(A) ::= table_alias(B). { PARSER_TRACE; A = B; } +correlation_or_recognition_opt(A) ::= AS table_alias(B). { PARSER_TRACE; A = B; } + +parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { PARSER_TRACE; A = B; } +parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { PARSER_TRACE; A = B; } + +/*7.10*********************************************** joined_table ************************************************************/ +joined_table(A) ::= + table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { PARSER_TRACE; A = createJoinTableNode(pCxt, C, B, D, E); } + +%type join_type { EJoinType } +%destructor join_type { PARSER_DESTRUCTOR_TRACE; } +join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } + +/*7.15*********************************************** query_specification *************************************************/ +query_specification(A) ::= + SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) + partition_by_clause_opt(F) twindow_clause_opt(G) + group_by_clause_opt(H) having_clause_opt(I). { + PARSER_TRACE; + A = createSelectStmt(pCxt, B, C, D); + A = addWhereClause(pCxt, A, E); + A = addPartitionByClause(pCxt, A, F); + A = addWindowClauseClause(pCxt, A, G); + A = addGroupByClause(pCxt, A, H); + A = addHavingClause(pCxt, A, I); + } + +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt { PARSER_DESTRUCTOR_TRACE; } +set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } +set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } +set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } + +%type select_list { SNodeList* } +%destructor select_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } +select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } + +%type select_sublist { SNodeList* } +%destructor select_sublist { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; } +select_item(A) ::= value_expression(B) NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } + +/*7.16*********************************************** query_expression ****************************************************/ +query_expression(A) ::= + query_expression_body(B) + order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + PARSER_TRACE; + A = addOrderByClause(pCxt, B, C); + A = addSlimitClause(pCxt, A, D); + A = addLimitClause(pCxt, A, E); + } + +query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } +query_expression_body(A) ::= + query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } +query_primary(A) ::= + NK_LP query_expression_body(B) + order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } + +slimit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +slimit_clause_opt(A) ::= SLIMIT signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, 0); } +slimit_clause_opt(A) ::= SLIMIT signed_integer(B) SOFFSET signed_integer(C). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } +slimit_clause_opt(A) ::= SLIMIT signed_integer(C) NK_COMMA signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } + +limit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +limit_clause_opt(A) ::= LIMIT signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, 0); } +limit_clause_opt(A) ::= LIMIT signed_integer(B) OFFSET signed_integer(C). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } +limit_clause_opt(A) ::= LIMIT signed_integer(C) NK_COMMA signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } + +/*7.18*********************************************** subquery ************************************************************/ +subquery(A) ::= NK_LR query_expression(B) NK_RP. { PARSER_TRACE; A = B; } + +/*8.1*********************************************** predicate ************************************************************/ +predicate ::= . + +/*8.21 todo *********************************************** search_condition ************************************************************/ +search_condition(A) ::= boolean_value_expression(B). { PARSER_TRACE; A = B; } + +/*10.10*********************************************** sort_specification_list ************************************************************/ +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= + sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= + value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } + +%type ordering_specification_opt EOrder +%destructor ordering_specification_opt { PARSER_DESTRUCTOR_TRACE; } +ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } + +%type null_ordering_opt ENullOrder +%destructor null_ordering_opt { PARSER_DESTRUCTOR_TRACE; } +null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } + +/************************************************ todo ************************************************************/ + +where_clause_opt ::= . + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +partition_by_clause_opt ::=. + +twindow_clause_opt ::= . + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_clause_opt ::= . + +having_clause_opt ::= . //////////////////////// value_function ///////////////////////////////// value_function ::= NK_ID NK_LP value_expression NK_RP. value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP. -//////////////////////// value_expression_primary ///////////////////////////////// -value_expression_primary ::= NK_LP value_expression NK_RP. -value_expression_primary ::= nonparenthesized_value_expression_primary. - -nonparenthesized_value_expression_primary ::= literal. -// ? -nonparenthesized_value_expression_primary ::= column_reference. - -literal ::= NK_LITERAL. - -column_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } -column_reference(A) ::= table_name(B) NK_DOT NK_ID(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } - //////////////////////// value_expression ///////////////////////////////// value_expression ::= common_value_expression. @@ -93,107 +293,4 @@ numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expr numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression. numeric_primary ::= value_expression_primary. -numeric_primary ::= value_function. - -//////////////////////// query_specification ///////////////////////////////// -query_specification(A) ::= SELECT set_quantifier_opt(B) select_list(C) from_clause(D). { PARSER_TRACE; A = createSelectStmt(pCxt, B, C, D); } - -%type set_quantifier_opt { bool } -%destructor set_quantifier_opt {} -set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } -set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } -set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } - -%type select_list { SNodeList* } -%destructor select_list { nodesDestroyNodeList($$); } -select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } -select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } - -%type select_sublist { SNodeList* } -%destructor select_sublist { nodesDestroyNodeList($$); } -select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } -select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } - -select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; } -select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } -select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } - -from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } - -//%type table_reference_list { SNodeList* } -//%destructor table_reference_list { nodesDestroyNodeList($$); } -table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } -//table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, B, C); } - -//table_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createRealTableNode(pCxt, ); } -table_reference(A) ::= table_factor(B). { PARSER_TRACE; A = B; } -//table_reference ::= joined_table. - -table_factor(A) ::= table_primary(B). { PARSER_TRACE; A = B; } - -table_primary(A) ::= table_name(B). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B); } -table_primary(A) ::= db_name(B) NK_DOT table_name(C). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C); } -table_primary ::= derived_table. - -derived_table ::= table_subquery. - -%type db_name { SToken } -db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -%type table_name { SToken } -table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } - -//////////////////////// subquery ///////////////////////////////// -subquery ::= NK_LR query_expression NK_RP. - -table_subquery ::= subquery. - -// query_expression -query_expression(A) ::= with_clause_opt query_expression_body(B) order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { - PARSER_TRACE; - addOrderByList(pCxt, B, C); - addSlimit(pCxt, B, D); - addLimit(pCxt, B, E); - A = B; - } - -// WITH AS -with_clause_opt ::= . {} - -query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } -query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } - -query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } -query_primary(A) ::= NK_LP query_expression_body(B) order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;} - -%type order_by_clause_opt { SNodeList* } -%destructor order_by_clause_opt { nodesDestroyNodeList($$); } -order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } - -slimit_clause_opt(A) ::= . { A = NULL; } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } - -limit_clause_opt(A) ::= . { A = NULL; } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } - -//////////////////////// sort_specification_list ///////////////////////////////// -%type sort_specification_list { SNodeList* } -%destructor sort_specification_list { nodesDestroyNodeList($$); } -sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } -sort_specification_list(A) ::= sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } - -sort_specification(A) ::= value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } - -%type ordering_specification_opt EOrder -%destructor ordering_specification_opt {} -ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } -ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } -ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } - -%type null_ordering_opt ENullOrder -%destructor null_ordering_opt {} -null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } -null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } -null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } +numeric_primary ::= value_function. \ No newline at end of file diff --git a/source/libs/parser/src/astCreateContext.c b/source/libs/parser/src/astCreateContext.c index 42c5257360..6e57683afd 100644 --- a/source/libs/parser/src/astCreateContext.c +++ b/source/libs/parser/src/astCreateContext.c @@ -16,10 +16,46 @@ #include "ttoken.h" #include "astCreateContext.h" -void* acquireRaii(SAstCreateContext* pCxt, void* p) { +typedef struct SResourceEntry { + EResourceType type; + void* res; +} SResourceEntry; + +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt) { + pCxt->pQueryCxt = pQueryCxt; + pCxt->notSupport = false; + pCxt->valid = true; + pCxt->pRootNode = NULL; + pCxt->pResourceHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (NULL == pCxt->pResourceHash) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +int32_t destroyAstCreateContext(SAstCreateContext* pCxt) { + SResourceEntry* item = taosHashIterate(pCxt->pResourceHash, NULL); + while (item) { + switch (item->type) { + case AST_CXT_RESOURCE_NODE: + nodesDestroyNode(item->res); + break; + case AST_CXT_RESOURCE_NODE_LIST: + nodesDestroyList(item->res); + break; + default: + tfree(item->res); + } + item = taosHashIterate(pCxt->pResourceHash, item); + } +} + +void* acquireRaii(SAstCreateContext* pCxt, EResourceType type, void* p) { if (NULL == p) { return NULL; } + SResourceEntry entry = { .type = type, .res = p }; + taosHashPut(pCxt->pResourceHash, &p, POINTER_BYTES, &entry, sizeof(SResourceEntry)); return p; } @@ -27,14 +63,6 @@ void* releaseRaii(SAstCreateContext* pCxt, void* p) { if (NULL == p) { return NULL; } + taosHashRemove(pCxt->pResourceHash, &p, POINTER_BYTES); return p; } - -int32_t createAstCreater(const SParseContext* pQueryCxt, SAstCreateContext* pCxt) { - -} - -int32_t destroyAstCreater(SAstCreateContext* pCxt) { - -} - diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index 337ae78d75..8c67fc1690 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -16,76 +16,216 @@ #include "astCreateFuncs.h" -#include "astCreateContext.h" +#define CHECK_OUT_OF_MEM(p) \ + do { \ + if (NULL == (p)) { \ + pCxt->valid = false; \ + return NULL; \ + } \ + } while (0) -bool checkTableName(const SToken* pTableName) { - printf("%p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); - return pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; +SToken nil_token = { .type = TK_NIL, .n = 0, .z = NULL }; + +static bool checkDbName(SAstCreateContext* pCxt, const SToken* pDbName) { + if (NULL == pDbName) { + return true; + } + pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false; + return pCxt->valid; } -SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { - +static bool checkTableName(SAstCreateContext* pCxt, const SToken* pTableName) { + if (NULL == pTableName) { + return true; + } + pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; + return pCxt->valid; } -SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { - -} - -SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { - -} - -SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { - -} - -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { - -} - -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { - +static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) { + if (NULL == pColumnName) { + return true; + } + pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false; + return pCxt->valid; } SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { + SNodeList* list = nodesMakeList(); + CHECK_OUT_OF_MEM(list); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE_LIST, nodesListAppend(list, releaseRaii(pCxt, pNode))); +} +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { + return nodesListAppend(pList, releaseRaii(pCxt, pNode)); +} + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { + if (!checkTableName(pCxt, pTableName) || !checkColumnName(pCxt, pColumnName)) { + return NULL; + } + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + CHECK_OUT_OF_MEM(col); + if (NULL != pTableName) { + strncpy(col->tableName, pTableName->z, pTableName->n); + } + strncpy(col->colName, pColumnName->z, pColumnName->n); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, col); +} + +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + // strncpy(col->colName, pColumnName->z, pColumnName->n); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, val); +} + +SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode) { + // todo +} + +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2) { + SLogicConditionNode* cond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->condType = type; + cond->pParameterList = nodesMakeList(); + nodesListAppend(cond->pParameterList, releaseRaii(pCxt, pParam1)); + nodesListAppend(cond->pParameterList, releaseRaii(pCxt, pParam2)); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, cond); +} + +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias) { + if (!checkDbName(pCxt, pDbName) || !checkTableName(pCxt, pTableName)) { + return NULL; + } + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + CHECK_OUT_OF_MEM(realTable); + if (NULL != pDbName) { + strncpy(realTable->dbName, pDbName->z, pDbName->n); + } + strncpy(realTable->table.tableName, pTableName->z, pTableName->n); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, realTable); +} + +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias) { + STempTableNode* tempTable = (STempTableNode*)nodesMakeNode(QUERY_NODE_TEMP_TABLE); + CHECK_OUT_OF_MEM(tempTable); + tempTable->pSubquery = pSubquery; + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, tempTable); +} + +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond) { + SJoinTableNode* joinTable = (SJoinTableNode*)nodesMakeNode(QUERY_NODE_JOIN_TABLE); + CHECK_OUT_OF_MEM(joinTable); + joinTable->joinType = type; + joinTable->pLeft = releaseRaii(pCxt, pLeft); + joinTable->pRight = releaseRaii(pCxt, pRight); + joinTable->pOnCond = pJoinCond; + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, joinTable); +} + +SNode* createLimitNode(SAstCreateContext* pCxt, SNode* pLimit, SNode* pOffset) { + SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + CHECK_OUT_OF_MEM(limitNode); + // limitNode->limit = limit; + // limitNode->offset = offset; + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, limitNode); } SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { - + SOrderByExprNode* orderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + CHECK_OUT_OF_MEM(orderByExpr); + orderByExpr->pExpr = releaseRaii(pCxt, pExpr); + orderByExpr->order = order; + orderByExpr->nullOrder = nullOrder; + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, orderByExpr); } -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName) { - SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); - if (NULL != pDbName) { - printf("DbName %p : %d, %d, %s\n", pDbName, pDbName->type, pDbName->n, pDbName->z); - strncpy(realTable->dbName, pDbName->z, pDbName->n); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { + strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); + return pNode; +} + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWhere = releaseRaii(pCxt, pWhere); } - printf("TableName %p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); - strncpy(realTable->table.tableName, pTableName->z, pTableName->n); - return acquireRaii(pCxt, realTable); + return pStmt; +} + +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pPartitionByList = releaseRaii(pCxt, pPartitionByList); + } + return pStmt; +} + +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWindow = releaseRaii(pCxt, pWindow); + } + return pStmt; +} + +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pGroupByList = releaseRaii(pCxt, pGroupByList); + } + return pStmt; +} + +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pHaving = releaseRaii(pCxt, pHaving); + } + return pStmt; +} + +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pOrderByList = releaseRaii(pCxt, pOrderByList); + } + return pStmt; +} + +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pSlimit = releaseRaii(pCxt, pSlimit); + } + return pStmt; +} + +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pLimit = releaseRaii(pCxt, pLimit); + } + return pStmt; } SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + CHECK_OUT_OF_MEM(select); select->isDistinct = isDistinct; if (NULL == pProjectionList) { select->isStar = true; } select->pProjectionList = releaseRaii(pCxt, pProjectionList); - printf("pTable = %p, name = %s\n", pTable, ((SRealTableNode*)pTable)->table.tableName); select->pFromTable = releaseRaii(pCxt, pTable); - return acquireRaii(pCxt, select); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, select); } SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) { - + SSetOperator* setOp = (SSetOperator*)nodesMakeNode(QUERY_NODE_SET_OPERATOR); + CHECK_OUT_OF_MEM(setOp); + setOp->opType = type; + setOp->pLeft = releaseRaii(pCxt, pLeft); + setOp->pRight = releaseRaii(pCxt, pRight); + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, setOp); } SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) { - -} - -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { - + SShowStmt* show = (SShowStmt*)nodesMakeNode(QUERY_NODE_SHOW_STMT); + CHECK_OUT_OF_MEM(show); + show->showType = type; + return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, show); } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index 7bf29c45f8..f5eedc9557 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -37,7 +37,8 @@ #include "ttokendef.h" #include "astCreateFuncs.h" -#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -101,18 +102,19 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 71 -#define YYACTIONTYPE unsigned char +#define YYNOCODE 98 +#define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - bool yy9; - SNodeList* yy30; - SToken yy67; - ENullOrder yy68; - EOrder yy108; - SNode* yy130; + SNodeList* yy40; + EOrder yy106; + EJoinType yy148; + SNode* yy168; + SToken yy169; + bool yy173; + ENullOrder yy193; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +129,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 63 -#define YYNRULE 68 -#define YYNTOKEN 35 -#define YY_MAX_SHIFT 62 -#define YY_MIN_SHIFTREDUCE 114 -#define YY_MAX_SHIFTREDUCE 181 -#define YY_ERROR_ACTION 182 -#define YY_ACCEPT_ACTION 183 -#define YY_NO_ACTION 184 -#define YY_MIN_REDUCE 185 -#define YY_MAX_REDUCE 252 +#define YYNSTATE 92 +#define YYNRULE 106 +#define YYNTOKEN 45 +#define YY_MAX_SHIFT 91 +#define YY_MIN_SHIFTREDUCE 169 +#define YY_MAX_SHIFTREDUCE 274 +#define YY_ERROR_ACTION 275 +#define YY_ACCEPT_ACTION 276 +#define YY_NO_ACTION 277 +#define YY_MIN_REDUCE 278 +#define YY_MAX_REDUCE 383 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -204,89 +206,142 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (229) +#define YY_ACTTAB_COUNT (464) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 30, 50, 30, 30, 30, 30, 55, 50, 30, 30, - /* 10 */ 10, 9, 29, 62, 51, 195, 30, 19, 30, 30, - /* 20 */ 30, 30, 57, 19, 30, 30, 30, 19, 30, 30, - /* 30 */ 30, 30, 57, 19, 30, 30, 30, 50, 30, 30, - /* 40 */ 30, 30, 55, 50, 30, 30, 39, 222, 155, 156, - /* 50 */ 23, 196, 12, 11, 10, 9, 25, 223, 30, 58, - /* 60 */ 30, 30, 30, 30, 57, 58, 30, 30, 30, 59, - /* 70 */ 30, 30, 30, 30, 57, 59, 30, 30, 30, 37, - /* 80 */ 30, 30, 30, 30, 57, 37, 30, 30, 204, 38, - /* 90 */ 5, 183, 60, 166, 158, 159, 121, 53, 200, 201, - /* 100 */ 202, 203, 54, 203, 203, 203, 246, 18, 246, 246, - /* 110 */ 246, 246, 57, 120, 246, 246, 245, 17, 245, 245, - /* 120 */ 245, 245, 57, 46, 245, 245, 35, 27, 35, 35, - /* 130 */ 35, 35, 57, 31, 35, 35, 36, 47, 36, 36, - /* 140 */ 36, 36, 57, 128, 36, 36, 242, 117, 242, 242, - /* 150 */ 242, 242, 57, 42, 241, 242, 241, 241, 241, 241, - /* 160 */ 57, 52, 212, 241, 14, 13, 122, 1, 212, 212, - /* 170 */ 38, 5, 14, 13, 166, 40, 224, 20, 38, 5, - /* 180 */ 15, 210, 166, 211, 21, 22, 17, 210, 210, 43, - /* 190 */ 160, 6, 7, 45, 41, 26, 207, 48, 24, 208, - /* 200 */ 136, 44, 205, 8, 189, 3, 142, 2, 16, 127, - /* 210 */ 147, 146, 32, 150, 49, 149, 4, 33, 180, 206, - /* 220 */ 28, 117, 34, 186, 56, 162, 161, 185, 61, + /* 0 */ 60, 60, 77, 60, 60, 60, 290, 80, 289, 47, + /* 10 */ 60, 60, 62, 60, 60, 60, 171, 172, 245, 246, + /* 20 */ 247, 70, 249, 6, 21, 20, 76, 33, 292, 295, + /* 30 */ 46, 11, 10, 9, 8, 31, 72, 318, 34, 22, + /* 40 */ 19, 42, 300, 75, 49, 74, 187, 60, 62, 60, + /* 50 */ 60, 60, 60, 179, 60, 60, 60, 22, 87, 289, + /* 60 */ 44, 60, 60, 37, 60, 60, 60, 18, 202, 63, + /* 70 */ 291, 295, 294, 291, 295, 73, 291, 295, 60, 60, + /* 80 */ 310, 60, 60, 60, 262, 80, 289, 5, 60, 60, + /* 90 */ 62, 60, 60, 60, 83, 341, 51, 50, 60, 37, + /* 100 */ 60, 60, 171, 79, 53, 15, 60, 60, 306, 60, + /* 110 */ 60, 60, 1, 87, 289, 319, 60, 60, 37, 60, + /* 120 */ 60, 60, 19, 304, 206, 60, 62, 60, 60, 60, + /* 130 */ 60, 306, 60, 60, 60, 202, 87, 289, 212, 60, + /* 140 */ 60, 81, 60, 60, 60, 71, 303, 61, 291, 295, + /* 150 */ 342, 205, 177, 60, 37, 60, 60, 60, 60, 306, + /* 160 */ 60, 60, 60, 32, 87, 289, 23, 60, 60, 88, + /* 170 */ 60, 60, 60, 214, 302, 40, 60, 81, 60, 60, + /* 180 */ 60, 60, 178, 60, 60, 60, 327, 87, 289, 327, + /* 190 */ 60, 60, 64, 60, 60, 60, 38, 236, 237, 326, + /* 200 */ 325, 9, 8, 325, 60, 88, 60, 60, 381, 381, + /* 210 */ 307, 381, 381, 381, 179, 87, 289, 14, 381, 381, + /* 220 */ 67, 381, 381, 381, 282, 28, 334, 60, 64, 60, + /* 230 */ 60, 380, 380, 59, 380, 380, 380, 29, 87, 289, + /* 240 */ 343, 380, 380, 91, 380, 380, 380, 282, 282, 333, + /* 250 */ 68, 282, 23, 338, 282, 381, 337, 381, 381, 65, + /* 260 */ 65, 40, 65, 65, 65, 86, 87, 289, 27, 65, + /* 270 */ 65, 30, 65, 65, 65, 282, 85, 69, 380, 48, + /* 280 */ 380, 380, 66, 66, 171, 66, 66, 66, 284, 87, + /* 290 */ 289, 283, 66, 66, 84, 66, 66, 66, 21, 20, + /* 300 */ 239, 240, 25, 7, 324, 286, 65, 52, 65, 65, + /* 310 */ 377, 377, 26, 377, 377, 377, 178, 87, 289, 312, + /* 320 */ 377, 377, 56, 377, 377, 377, 55, 57, 58, 66, + /* 330 */ 213, 66, 66, 376, 376, 3, 376, 376, 376, 20, + /* 340 */ 87, 289, 200, 376, 376, 199, 376, 376, 376, 43, + /* 350 */ 17, 16, 285, 36, 263, 230, 24, 377, 13, 12, + /* 360 */ 377, 4, 219, 250, 171, 172, 245, 246, 247, 70, + /* 370 */ 249, 6, 2, 279, 54, 276, 89, 278, 90, 82, + /* 380 */ 376, 277, 277, 376, 277, 277, 277, 277, 19, 277, + /* 390 */ 13, 12, 207, 277, 277, 277, 171, 172, 245, 246, + /* 400 */ 247, 70, 249, 6, 277, 327, 277, 277, 327, 277, + /* 410 */ 277, 277, 76, 33, 277, 39, 277, 277, 39, 325, + /* 420 */ 19, 31, 325, 76, 33, 277, 78, 41, 300, 301, + /* 430 */ 277, 300, 31, 277, 277, 277, 277, 277, 35, 300, + /* 440 */ 301, 277, 300, 277, 76, 33, 277, 277, 277, 277, + /* 450 */ 277, 277, 277, 31, 277, 277, 277, 277, 277, 45, + /* 460 */ 300, 301, 277, 300, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - /* 10 */ 8, 9, 49, 10, 51, 52, 37, 38, 39, 40, - /* 20 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, - /* 30 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, - /* 40 */ 41, 42, 43, 44, 45, 46, 67, 68, 30, 31, - /* 50 */ 69, 52, 6, 7, 8, 9, 63, 68, 37, 38, - /* 60 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, - /* 70 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, - /* 80 */ 39, 40, 41, 42, 43, 44, 45, 46, 43, 12, - /* 90 */ 13, 35, 36, 16, 33, 34, 2, 12, 53, 54, - /* 100 */ 55, 56, 57, 58, 59, 60, 37, 22, 39, 40, - /* 110 */ 41, 42, 43, 19, 45, 46, 37, 61, 39, 40, - /* 120 */ 41, 42, 43, 15, 45, 46, 37, 63, 39, 40, - /* 130 */ 41, 42, 43, 32, 45, 46, 37, 29, 39, 40, - /* 140 */ 41, 42, 43, 8, 45, 46, 37, 12, 39, 40, - /* 150 */ 41, 42, 43, 1, 37, 46, 39, 40, 41, 42, - /* 160 */ 43, 36, 47, 46, 6, 7, 8, 48, 47, 47, - /* 170 */ 12, 13, 6, 7, 16, 23, 70, 62, 12, 13, - /* 180 */ 13, 66, 16, 62, 62, 18, 61, 66, 66, 15, - /* 190 */ 14, 15, 13, 25, 64, 64, 17, 28, 65, 65, - /* 200 */ 12, 27, 43, 21, 50, 15, 14, 24, 2, 12, - /* 210 */ 26, 26, 26, 26, 20, 26, 15, 26, 14, 17, - /* 220 */ 17, 12, 17, 0, 17, 14, 14, 0, 11, 71, - /* 230 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 240 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 250 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 260 */ 71, 71, 71, 71, + /* 0 */ 47, 48, 73, 50, 51, 52, 55, 54, 55, 92, + /* 10 */ 57, 58, 59, 60, 61, 62, 15, 16, 17, 18, + /* 20 */ 19, 20, 21, 22, 1, 2, 53, 54, 64, 65, + /* 30 */ 77, 9, 10, 11, 12, 62, 83, 84, 3, 22, + /* 40 */ 39, 68, 69, 70, 86, 72, 23, 94, 95, 96, + /* 50 */ 97, 47, 48, 20, 50, 51, 52, 22, 54, 55, + /* 60 */ 27, 57, 58, 59, 60, 61, 62, 29, 30, 63, + /* 70 */ 64, 65, 63, 64, 65, 63, 64, 65, 47, 48, + /* 80 */ 74, 50, 51, 52, 23, 54, 55, 26, 57, 58, + /* 90 */ 59, 60, 61, 62, 90, 91, 9, 10, 94, 95, + /* 100 */ 96, 97, 15, 20, 86, 22, 47, 48, 56, 50, + /* 110 */ 51, 52, 76, 54, 55, 84, 57, 58, 59, 60, + /* 120 */ 61, 62, 39, 71, 5, 94, 95, 96, 97, 47, + /* 130 */ 48, 56, 50, 51, 52, 30, 54, 55, 20, 57, + /* 140 */ 58, 59, 60, 61, 62, 27, 71, 63, 64, 65, + /* 150 */ 91, 32, 20, 94, 95, 96, 97, 47, 48, 56, + /* 160 */ 50, 51, 52, 54, 54, 55, 22, 57, 58, 59, + /* 170 */ 60, 61, 62, 11, 71, 31, 94, 95, 96, 97, + /* 180 */ 47, 48, 20, 50, 51, 52, 75, 54, 55, 75, + /* 190 */ 57, 58, 59, 60, 61, 62, 85, 40, 41, 85, + /* 200 */ 89, 11, 12, 89, 94, 95, 96, 97, 47, 48, + /* 210 */ 56, 50, 51, 52, 20, 54, 55, 25, 57, 58, + /* 220 */ 42, 60, 61, 62, 47, 26, 49, 94, 95, 96, + /* 230 */ 97, 47, 48, 66, 50, 51, 52, 38, 54, 55, + /* 240 */ 93, 57, 58, 13, 60, 61, 62, 47, 47, 49, + /* 250 */ 49, 47, 22, 49, 47, 94, 49, 96, 97, 47, + /* 260 */ 48, 31, 50, 51, 52, 4, 54, 55, 35, 57, + /* 270 */ 58, 37, 60, 61, 62, 47, 87, 49, 94, 88, + /* 280 */ 96, 97, 47, 48, 15, 50, 51, 52, 47, 54, + /* 290 */ 55, 47, 57, 58, 33, 60, 61, 62, 1, 2, + /* 300 */ 43, 44, 26, 22, 88, 24, 94, 87, 96, 97, + /* 310 */ 47, 48, 36, 50, 51, 52, 20, 54, 55, 82, + /* 320 */ 57, 58, 80, 60, 61, 62, 81, 79, 78, 94, + /* 330 */ 20, 96, 97, 47, 48, 26, 50, 51, 52, 2, + /* 340 */ 54, 55, 23, 57, 58, 23, 60, 61, 62, 24, + /* 350 */ 26, 28, 24, 24, 23, 23, 5, 94, 9, 10, + /* 360 */ 97, 26, 23, 23, 15, 16, 17, 18, 19, 20, + /* 370 */ 21, 22, 34, 0, 24, 45, 46, 0, 14, 46, + /* 380 */ 94, 98, 98, 97, 98, 98, 98, 98, 39, 98, + /* 390 */ 9, 10, 11, 98, 98, 98, 15, 16, 17, 18, + /* 400 */ 19, 20, 21, 22, 98, 75, 98, 98, 75, 98, + /* 410 */ 98, 98, 53, 54, 98, 85, 98, 98, 85, 89, + /* 420 */ 39, 62, 89, 53, 54, 98, 67, 68, 69, 70, + /* 430 */ 98, 72, 62, 98, 98, 98, 98, 98, 68, 69, + /* 440 */ 70, 98, 72, 98, 53, 54, 98, 98, 98, 98, + /* 450 */ 98, 98, 98, 62, 98, 98, 98, 98, 98, 68, + /* 460 */ 69, 70, 98, 72, }; -#define YY_SHIFT_COUNT (62) +#define YY_SHIFT_COUNT (91) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (227) +#define YY_SHIFT_MAX (381) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 3, 158, 166, 166, 166, 166, 166, 166, 85, 166, - /* 10 */ 166, 166, 166, 77, 77, 167, 167, 167, 229, 18, - /* 20 */ 152, 152, 94, 101, 168, 169, 169, 168, 188, 182, - /* 30 */ 46, 61, 174, 108, 135, 2, 2, 176, 179, 190, - /* 40 */ 183, 192, 206, 184, 185, 186, 187, 189, 191, 197, - /* 50 */ 194, 201, 204, 202, 203, 205, 209, 207, 211, 212, - /* 60 */ 223, 227, 217, + /* 0 */ 230, 381, 349, 349, 349, 349, 349, 349, 349, 349, + /* 10 */ 349, 349, 1, 1, 83, 83, 83, 83, 35, 144, + /* 20 */ 35, 35, 35, 144, 144, 87, 87, 87, 87, 87, + /* 30 */ 87, 33, 33, 33, 17, 38, 162, 157, 261, 261, + /* 40 */ 119, 105, 105, 132, 194, 105, 192, 178, 233, 234, + /* 50 */ 269, 269, 234, 233, 296, 464, 464, 464, 464, 464, + /* 60 */ 22, 23, 118, 297, 61, 190, 190, 257, 276, 199, + /* 70 */ 281, 310, 309, 337, 319, 322, 325, 323, 324, 328, + /* 80 */ 329, 331, 332, 335, 338, 339, 351, 350, 340, 373, + /* 90 */ 377, 364, }; -#define YY_REDUCE_COUNT (29) -#define YY_REDUCE_MIN (-37) -#define YY_REDUCE_MAX (159) +#define YY_REDUCE_COUNT (59) +#define YY_REDUCE_MIN (-83) +#define YY_REDUCE_MAX (391) static const short yy_reduce_ofst[] = { - /* 0 */ 56, -37, -21, -11, -1, 21, 31, 41, 45, 69, - /* 10 */ 79, 89, 99, 109, 117, 115, 121, 122, 125, -19, - /* 20 */ -7, 64, 119, 106, 130, 133, 134, 131, 159, 154, + /* 0 */ 330, -47, 4, 31, 59, 82, 110, 133, 161, 184, + /* 10 */ 212, 235, 263, 286, 359, -27, 370, 391, 6, 333, + /* 20 */ 9, 12, 84, 111, 114, 177, 200, 201, 204, 207, + /* 30 */ 228, 52, 75, 103, -36, -71, -49, -83, -42, 18, + /* 40 */ 36, -71, -71, 109, 154, -71, 167, 147, 189, 191, + /* 50 */ 241, 244, 216, 220, -49, 237, 245, 242, 248, 250, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 209, 182, 182, 182, 182, 182, 182, 182, 182, 182, - /* 10 */ 182, 182, 182, 182, 182, 182, 182, 182, 209, 225, - /* 20 */ 214, 214, 190, 228, 216, 219, 219, 216, 182, 182, - /* 30 */ 239, 182, 182, 182, 182, 244, 243, 182, 187, 215, - /* 40 */ 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, - /* 50 */ 197, 194, 182, 207, 182, 182, 182, 182, 182, 182, - /* 60 */ 182, 182, 182, + /* 0 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 10 */ 275, 275, 275, 275, 275, 275, 275, 275, 365, 275, + /* 20 */ 365, 365, 365, 275, 275, 275, 275, 275, 275, 275, + /* 30 */ 275, 305, 305, 305, 365, 275, 275, 344, 329, 329, + /* 40 */ 313, 298, 275, 275, 275, 299, 275, 347, 331, 335, + /* 50 */ 275, 275, 335, 331, 275, 370, 369, 368, 367, 366, + /* 60 */ 374, 275, 320, 340, 275, 379, 378, 275, 332, 336, + /* 70 */ 287, 275, 317, 293, 364, 301, 275, 275, 297, 286, + /* 80 */ 275, 275, 275, 330, 275, 275, 275, 275, 275, 275, + /* 90 */ 275, 275, }; /********** End of lemon-generated parsing tables *****************************/ @@ -394,76 +449,103 @@ void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ ** are required. The following table supplies these names */ static const char *const yyTokenName[] = { /* 0 */ "$", - /* 1 */ "UNION", - /* 2 */ "ALL", - /* 3 */ "MINUS", - /* 4 */ "EXCEPT", - /* 5 */ "INTERSECT", - /* 6 */ "NK_PLUS", - /* 7 */ "NK_MINUS", - /* 8 */ "NK_STAR", - /* 9 */ "NK_SLASH", - /* 10 */ "SHOW", - /* 11 */ "DATABASES", - /* 12 */ "NK_ID", - /* 13 */ "NK_LP", - /* 14 */ "NK_RP", - /* 15 */ "NK_COMMA", - /* 16 */ "NK_LITERAL", - /* 17 */ "NK_DOT", - /* 18 */ "SELECT", - /* 19 */ "DISTINCT", - /* 20 */ "AS", - /* 21 */ "FROM", - /* 22 */ "NK_LR", - /* 23 */ "ORDER", - /* 24 */ "BY", - /* 25 */ "SLIMIT", - /* 26 */ "NK_INTEGER", - /* 27 */ "SOFFSET", - /* 28 */ "LIMIT", - /* 29 */ "OFFSET", - /* 30 */ "ASC", - /* 31 */ "DESC", - /* 32 */ "NULLS", - /* 33 */ "FIRST", - /* 34 */ "LAST", - /* 35 */ "cmd", - /* 36 */ "query_expression", - /* 37 */ "value_function", - /* 38 */ "value_expression", - /* 39 */ "value_expression_primary", - /* 40 */ "nonparenthesized_value_expression_primary", - /* 41 */ "literal", - /* 42 */ "column_reference", - /* 43 */ "table_name", - /* 44 */ "common_value_expression", - /* 45 */ "numeric_value_expression", - /* 46 */ "numeric_primary", - /* 47 */ "query_specification", - /* 48 */ "set_quantifier_opt", - /* 49 */ "select_list", - /* 50 */ "from_clause", - /* 51 */ "select_sublist", - /* 52 */ "select_item", - /* 53 */ "table_reference_list", - /* 54 */ "table_reference", - /* 55 */ "table_factor", - /* 56 */ "table_primary", - /* 57 */ "db_name", - /* 58 */ "derived_table", - /* 59 */ "table_subquery", - /* 60 */ "subquery", - /* 61 */ "with_clause_opt", - /* 62 */ "query_expression_body", - /* 63 */ "order_by_clause_opt", - /* 64 */ "slimit_clause_opt", - /* 65 */ "limit_clause_opt", - /* 66 */ "query_primary", - /* 67 */ "sort_specification_list", - /* 68 */ "sort_specification", - /* 69 */ "ordering_specification_opt", - /* 70 */ "null_ordering_opt", + /* 1 */ "OR", + /* 2 */ "AND", + /* 3 */ "NOT", + /* 4 */ "UNION", + /* 5 */ "ALL", + /* 6 */ "MINUS", + /* 7 */ "EXCEPT", + /* 8 */ "INTERSECT", + /* 9 */ "NK_PLUS", + /* 10 */ "NK_MINUS", + /* 11 */ "NK_STAR", + /* 12 */ "NK_SLASH", + /* 13 */ "SHOW", + /* 14 */ "DATABASES", + /* 15 */ "NK_INTEGER", + /* 16 */ "NK_FLOAT", + /* 17 */ "NK_STRING", + /* 18 */ "NK_BOOL", + /* 19 */ "NK_NOW", + /* 20 */ "NK_ID", + /* 21 */ "NK_QUESTION", + /* 22 */ "NK_LP", + /* 23 */ "NK_RP", + /* 24 */ "NK_DOT", + /* 25 */ "FROM", + /* 26 */ "NK_COMMA", + /* 27 */ "AS", + /* 28 */ "JOIN", + /* 29 */ "ON", + /* 30 */ "INNER", + /* 31 */ "SELECT", + /* 32 */ "DISTINCT", + /* 33 */ "ORDER", + /* 34 */ "BY", + /* 35 */ "SLIMIT", + /* 36 */ "SOFFSET", + /* 37 */ "LIMIT", + /* 38 */ "OFFSET", + /* 39 */ "NK_LR", + /* 40 */ "ASC", + /* 41 */ "DESC", + /* 42 */ "NULLS", + /* 43 */ "FIRST", + /* 44 */ "LAST", + /* 45 */ "cmd", + /* 46 */ "query_expression", + /* 47 */ "unsigned_integer", + /* 48 */ "unsigned_approximate_numeric", + /* 49 */ "signed_integer", + /* 50 */ "unsigned_literal", + /* 51 */ "unsigned_numeric_literal", + /* 52 */ "general_literal", + /* 53 */ "db_name", + /* 54 */ "table_name", + /* 55 */ "column_name", + /* 56 */ "table_alias", + /* 57 */ "unsigned_value_specification", + /* 58 */ "value_expression_primary", + /* 59 */ "value_expression", + /* 60 */ "nonparenthesized_value_expression_primary", + /* 61 */ "column_reference", + /* 62 */ "subquery", + /* 63 */ "boolean_value_expression", + /* 64 */ "boolean_primary", + /* 65 */ "predicate", + /* 66 */ "from_clause", + /* 67 */ "table_reference_list", + /* 68 */ "table_reference", + /* 69 */ "table_primary", + /* 70 */ "joined_table", + /* 71 */ "correlation_or_recognition_opt", + /* 72 */ "parenthesized_joined_table", + /* 73 */ "join_type", + /* 74 */ "search_condition", + /* 75 */ "query_specification", + /* 76 */ "set_quantifier_opt", + /* 77 */ "select_list", + /* 78 */ "where_clause_opt", + /* 79 */ "partition_by_clause_opt", + /* 80 */ "twindow_clause_opt", + /* 81 */ "group_by_clause_opt", + /* 82 */ "having_clause_opt", + /* 83 */ "select_sublist", + /* 84 */ "select_item", + /* 85 */ "query_expression_body", + /* 86 */ "order_by_clause_opt", + /* 87 */ "slimit_clause_opt", + /* 88 */ "limit_clause_opt", + /* 89 */ "query_primary", + /* 90 */ "sort_specification_list", + /* 91 */ "sort_specification", + /* 92 */ "ordering_specification_opt", + /* 93 */ "null_ordering_opt", + /* 94 */ "value_function", + /* 95 */ "common_value_expression", + /* 96 */ "numeric_value_expression", + /* 97 */ "numeric_primary", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -473,72 +555,110 @@ static const char *const yyTokenName[] = { static const char *const yyRuleName[] = { /* 0 */ "cmd ::= SHOW DATABASES", /* 1 */ "cmd ::= query_expression", - /* 2 */ "column_reference ::= NK_ID", - /* 3 */ "column_reference ::= table_name NK_DOT NK_ID", - /* 4 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause", - /* 5 */ "set_quantifier_opt ::=", - /* 6 */ "set_quantifier_opt ::= DISTINCT", - /* 7 */ "set_quantifier_opt ::= ALL", - /* 8 */ "select_list ::= NK_STAR", - /* 9 */ "select_list ::= select_sublist", - /* 10 */ "select_sublist ::= select_item", - /* 11 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 12 */ "select_item ::= value_expression", - /* 13 */ "select_item ::= value_expression AS NK_ID", - /* 14 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 15 */ "from_clause ::= FROM table_reference_list", - /* 16 */ "table_reference_list ::= table_reference", - /* 17 */ "table_reference ::= table_factor", - /* 18 */ "table_factor ::= table_primary", - /* 19 */ "table_primary ::= table_name", - /* 20 */ "table_primary ::= db_name NK_DOT table_name", - /* 21 */ "db_name ::= NK_ID", - /* 22 */ "table_name ::= NK_ID", - /* 23 */ "query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 24 */ "with_clause_opt ::=", - /* 25 */ "query_expression_body ::= query_primary", - /* 26 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 27 */ "query_primary ::= query_specification", - /* 28 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", - /* 29 */ "order_by_clause_opt ::=", - /* 30 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 31 */ "slimit_clause_opt ::=", - /* 32 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 33 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 34 */ "limit_clause_opt ::=", - /* 35 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 36 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 37 */ "sort_specification_list ::= sort_specification", - /* 38 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 39 */ "sort_specification ::= value_expression ordering_specification_opt null_ordering_opt", - /* 40 */ "ordering_specification_opt ::=", - /* 41 */ "ordering_specification_opt ::= ASC", - /* 42 */ "ordering_specification_opt ::= DESC", - /* 43 */ "null_ordering_opt ::=", - /* 44 */ "null_ordering_opt ::= NULLS FIRST", - /* 45 */ "null_ordering_opt ::= NULLS LAST", - /* 46 */ "value_function ::= NK_ID NK_LP value_expression NK_RP", - /* 47 */ "value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP", - /* 48 */ "value_expression_primary ::= NK_LP value_expression NK_RP", - /* 49 */ "value_expression_primary ::= nonparenthesized_value_expression_primary", - /* 50 */ "nonparenthesized_value_expression_primary ::= literal", - /* 51 */ "nonparenthesized_value_expression_primary ::= column_reference", - /* 52 */ "literal ::= NK_LITERAL", - /* 53 */ "value_expression ::= common_value_expression", - /* 54 */ "common_value_expression ::= numeric_value_expression", - /* 55 */ "numeric_value_expression ::= numeric_primary", - /* 56 */ "numeric_value_expression ::= NK_PLUS numeric_primary", - /* 57 */ "numeric_value_expression ::= NK_MINUS numeric_primary", - /* 58 */ "numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression", - /* 59 */ "numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression", - /* 60 */ "numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression", - /* 61 */ "numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression", - /* 62 */ "numeric_primary ::= value_expression_primary", - /* 63 */ "numeric_primary ::= value_function", - /* 64 */ "table_primary ::= derived_table", - /* 65 */ "derived_table ::= table_subquery", - /* 66 */ "subquery ::= NK_LR query_expression NK_RP", - /* 67 */ "table_subquery ::= subquery", + /* 2 */ "unsigned_integer ::= NK_INTEGER", + /* 3 */ "unsigned_approximate_numeric ::= NK_FLOAT", + /* 4 */ "signed_integer ::= unsigned_integer", + /* 5 */ "signed_integer ::= NK_PLUS unsigned_integer", + /* 6 */ "signed_integer ::= NK_MINUS unsigned_integer", + /* 7 */ "db_name ::= NK_ID", + /* 8 */ "table_name ::= NK_ID", + /* 9 */ "column_name ::= NK_ID", + /* 10 */ "table_alias ::= NK_ID", + /* 11 */ "column_reference ::= column_name", + /* 12 */ "column_reference ::= table_name NK_DOT column_name", + /* 13 */ "boolean_value_expression ::= boolean_primary", + /* 14 */ "boolean_value_expression ::= NOT boolean_primary", + /* 15 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 16 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 17 */ "boolean_primary ::= predicate", + /* 18 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 19 */ "from_clause ::= FROM table_reference_list", + /* 20 */ "table_reference_list ::= table_reference", + /* 21 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 22 */ "table_reference ::= table_primary", + /* 23 */ "table_reference ::= joined_table", + /* 24 */ "table_primary ::= table_name correlation_or_recognition_opt", + /* 25 */ "table_primary ::= db_name NK_DOT table_name correlation_or_recognition_opt", + /* 26 */ "table_primary ::= subquery correlation_or_recognition_opt", + /* 27 */ "correlation_or_recognition_opt ::=", + /* 28 */ "correlation_or_recognition_opt ::= table_alias", + /* 29 */ "correlation_or_recognition_opt ::= AS table_alias", + /* 30 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 31 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 32 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 33 */ "join_type ::= INNER", + /* 34 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 35 */ "set_quantifier_opt ::=", + /* 36 */ "set_quantifier_opt ::= DISTINCT", + /* 37 */ "set_quantifier_opt ::= ALL", + /* 38 */ "select_list ::= NK_STAR", + /* 39 */ "select_list ::= select_sublist", + /* 40 */ "select_sublist ::= select_item", + /* 41 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 42 */ "select_item ::= value_expression", + /* 43 */ "select_item ::= value_expression NK_ID", + /* 44 */ "select_item ::= value_expression AS NK_ID", + /* 45 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 46 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 47 */ "query_expression_body ::= query_primary", + /* 48 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 49 */ "query_primary ::= query_specification", + /* 50 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", + /* 51 */ "order_by_clause_opt ::=", + /* 52 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 53 */ "slimit_clause_opt ::=", + /* 54 */ "slimit_clause_opt ::= SLIMIT signed_integer", + /* 55 */ "slimit_clause_opt ::= SLIMIT signed_integer SOFFSET signed_integer", + /* 56 */ "slimit_clause_opt ::= SLIMIT signed_integer NK_COMMA signed_integer", + /* 57 */ "limit_clause_opt ::=", + /* 58 */ "limit_clause_opt ::= LIMIT signed_integer", + /* 59 */ "limit_clause_opt ::= LIMIT signed_integer OFFSET signed_integer", + /* 60 */ "limit_clause_opt ::= LIMIT signed_integer NK_COMMA signed_integer", + /* 61 */ "subquery ::= NK_LR query_expression NK_RP", + /* 62 */ "search_condition ::= boolean_value_expression", + /* 63 */ "sort_specification_list ::= sort_specification", + /* 64 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 65 */ "sort_specification ::= value_expression ordering_specification_opt null_ordering_opt", + /* 66 */ "ordering_specification_opt ::=", + /* 67 */ "ordering_specification_opt ::= ASC", + /* 68 */ "ordering_specification_opt ::= DESC", + /* 69 */ "null_ordering_opt ::=", + /* 70 */ "null_ordering_opt ::= NULLS FIRST", + /* 71 */ "null_ordering_opt ::= NULLS LAST", + /* 72 */ "unsigned_literal ::= unsigned_numeric_literal", + /* 73 */ "unsigned_literal ::= general_literal", + /* 74 */ "unsigned_numeric_literal ::= unsigned_integer", + /* 75 */ "unsigned_numeric_literal ::= unsigned_approximate_numeric", + /* 76 */ "general_literal ::= NK_STRING", + /* 77 */ "general_literal ::= NK_BOOL", + /* 78 */ "general_literal ::= NK_NOW", + /* 79 */ "unsigned_value_specification ::= unsigned_literal", + /* 80 */ "unsigned_value_specification ::= NK_QUESTION", + /* 81 */ "value_expression_primary ::= NK_LP value_expression NK_RP", + /* 82 */ "value_expression_primary ::= nonparenthesized_value_expression_primary", + /* 83 */ "nonparenthesized_value_expression_primary ::= unsigned_value_specification", + /* 84 */ "nonparenthesized_value_expression_primary ::= column_reference", + /* 85 */ "nonparenthesized_value_expression_primary ::= subquery", + /* 86 */ "table_primary ::= parenthesized_joined_table", + /* 87 */ "predicate ::=", + /* 88 */ "where_clause_opt ::=", + /* 89 */ "partition_by_clause_opt ::=", + /* 90 */ "twindow_clause_opt ::=", + /* 91 */ "group_by_clause_opt ::=", + /* 92 */ "having_clause_opt ::=", + /* 93 */ "value_function ::= NK_ID NK_LP value_expression NK_RP", + /* 94 */ "value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP", + /* 95 */ "value_expression ::= common_value_expression", + /* 96 */ "common_value_expression ::= numeric_value_expression", + /* 97 */ "numeric_value_expression ::= numeric_primary", + /* 98 */ "numeric_value_expression ::= NK_PLUS numeric_primary", + /* 99 */ "numeric_value_expression ::= NK_MINUS numeric_primary", + /* 100 */ "numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression", + /* 101 */ "numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression", + /* 102 */ "numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression", + /* 103 */ "numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression", + /* 104 */ "numeric_primary ::= value_expression_primary", + /* 105 */ "numeric_primary ::= value_function", }; #endif /* NDEBUG */ @@ -665,60 +785,85 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 35: /* cmd */ - case 36: /* query_expression */ - case 37: /* value_function */ - case 38: /* value_expression */ - case 39: /* value_expression_primary */ - case 40: /* nonparenthesized_value_expression_primary */ - case 41: /* literal */ - case 42: /* column_reference */ - case 43: /* table_name */ - case 44: /* common_value_expression */ - case 45: /* numeric_value_expression */ - case 46: /* numeric_primary */ - case 47: /* query_specification */ - case 50: /* from_clause */ - case 52: /* select_item */ - case 53: /* table_reference_list */ - case 54: /* table_reference */ - case 55: /* table_factor */ - case 56: /* table_primary */ - case 57: /* db_name */ - case 58: /* derived_table */ - case 59: /* table_subquery */ - case 60: /* subquery */ - case 61: /* with_clause_opt */ - case 62: /* query_expression_body */ - case 64: /* slimit_clause_opt */ - case 65: /* limit_clause_opt */ - case 66: /* query_primary */ - case 68: /* sort_specification */ + case 45: /* cmd */ + case 46: /* query_expression */ + case 47: /* unsigned_integer */ + case 48: /* unsigned_approximate_numeric */ + case 49: /* signed_integer */ + case 50: /* unsigned_literal */ + case 51: /* unsigned_numeric_literal */ + case 52: /* general_literal */ + case 57: /* unsigned_value_specification */ + case 58: /* value_expression_primary */ + case 59: /* value_expression */ + case 60: /* nonparenthesized_value_expression_primary */ + case 61: /* column_reference */ + case 62: /* subquery */ + case 63: /* boolean_value_expression */ + case 64: /* boolean_primary */ + case 65: /* predicate */ + case 66: /* from_clause */ + case 67: /* table_reference_list */ + case 68: /* table_reference */ + case 69: /* table_primary */ + case 70: /* joined_table */ + case 72: /* parenthesized_joined_table */ + case 74: /* search_condition */ + case 75: /* query_specification */ + case 78: /* where_clause_opt */ + case 80: /* twindow_clause_opt */ + case 82: /* having_clause_opt */ + case 84: /* select_item */ + case 85: /* query_expression_body */ + case 87: /* slimit_clause_opt */ + case 88: /* limit_clause_opt */ + case 89: /* query_primary */ + case 91: /* sort_specification */ + case 94: /* value_function */ + case 95: /* common_value_expression */ + case 96: /* numeric_value_expression */ + case 97: /* numeric_primary */ { - nodesDestroyNode((yypminor->yy130)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy168)); } break; - case 48: /* set_quantifier_opt */ + case 53: /* db_name */ + case 54: /* table_name */ + case 55: /* column_name */ + case 56: /* table_alias */ + case 71: /* correlation_or_recognition_opt */ { - + PARSER_DESTRUCTOR_TRACE; } break; - case 49: /* select_list */ - case 51: /* select_sublist */ - case 63: /* order_by_clause_opt */ - case 67: /* sort_specification_list */ + case 73: /* join_type */ { - nodesDestroyNodeList((yypminor->yy30)); + PARSER_DESTRUCTOR_TRACE; } break; - case 69: /* ordering_specification_opt */ + case 76: /* set_quantifier_opt */ { - + PARSER_DESTRUCTOR_TRACE; } break; - case 70: /* null_ordering_opt */ + case 77: /* select_list */ + case 79: /* partition_by_clause_opt */ + case 81: /* group_by_clause_opt */ + case 83: /* select_sublist */ + case 86: /* order_by_clause_opt */ + case 90: /* sort_specification_list */ { - + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy40)); +} + break; + case 92: /* ordering_specification_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 93: /* null_ordering_opt */ +{ + PARSER_DESTRUCTOR_TRACE; } break; /********* End destructor definitions *****************************************/ @@ -1015,74 +1160,112 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 35, -2 }, /* (0) cmd ::= SHOW DATABASES */ - { 35, -1 }, /* (1) cmd ::= query_expression */ - { 42, -1 }, /* (2) column_reference ::= NK_ID */ - { 42, -3 }, /* (3) column_reference ::= table_name NK_DOT NK_ID */ - { 47, -4 }, /* (4) query_specification ::= SELECT set_quantifier_opt select_list from_clause */ - { 48, 0 }, /* (5) set_quantifier_opt ::= */ - { 48, -1 }, /* (6) set_quantifier_opt ::= DISTINCT */ - { 48, -1 }, /* (7) set_quantifier_opt ::= ALL */ - { 49, -1 }, /* (8) select_list ::= NK_STAR */ - { 49, -1 }, /* (9) select_list ::= select_sublist */ - { 51, -1 }, /* (10) select_sublist ::= select_item */ - { 51, -3 }, /* (11) select_sublist ::= select_sublist NK_COMMA select_item */ - { 52, -1 }, /* (12) select_item ::= value_expression */ - { 52, -3 }, /* (13) select_item ::= value_expression AS NK_ID */ - { 52, -3 }, /* (14) select_item ::= table_name NK_DOT NK_STAR */ - { 50, -2 }, /* (15) from_clause ::= FROM table_reference_list */ - { 53, -1 }, /* (16) table_reference_list ::= table_reference */ - { 54, -1 }, /* (17) table_reference ::= table_factor */ - { 55, -1 }, /* (18) table_factor ::= table_primary */ - { 56, -1 }, /* (19) table_primary ::= table_name */ - { 56, -3 }, /* (20) table_primary ::= db_name NK_DOT table_name */ - { 57, -1 }, /* (21) db_name ::= NK_ID */ - { 43, -1 }, /* (22) table_name ::= NK_ID */ - { 36, -5 }, /* (23) query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 61, 0 }, /* (24) with_clause_opt ::= */ - { 62, -1 }, /* (25) query_expression_body ::= query_primary */ - { 62, -4 }, /* (26) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 66, -1 }, /* (27) query_primary ::= query_specification */ - { 66, -6 }, /* (28) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ - { 63, 0 }, /* (29) order_by_clause_opt ::= */ - { 63, -3 }, /* (30) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 64, 0 }, /* (31) slimit_clause_opt ::= */ - { 64, -4 }, /* (32) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 64, -4 }, /* (33) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 65, 0 }, /* (34) limit_clause_opt ::= */ - { 65, -4 }, /* (35) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 65, -4 }, /* (36) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 67, -1 }, /* (37) sort_specification_list ::= sort_specification */ - { 67, -3 }, /* (38) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 68, -3 }, /* (39) sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ - { 69, 0 }, /* (40) ordering_specification_opt ::= */ - { 69, -1 }, /* (41) ordering_specification_opt ::= ASC */ - { 69, -1 }, /* (42) ordering_specification_opt ::= DESC */ - { 70, 0 }, /* (43) null_ordering_opt ::= */ - { 70, -2 }, /* (44) null_ordering_opt ::= NULLS FIRST */ - { 70, -2 }, /* (45) null_ordering_opt ::= NULLS LAST */ - { 37, -4 }, /* (46) value_function ::= NK_ID NK_LP value_expression NK_RP */ - { 37, -6 }, /* (47) value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ - { 39, -3 }, /* (48) value_expression_primary ::= NK_LP value_expression NK_RP */ - { 39, -1 }, /* (49) value_expression_primary ::= nonparenthesized_value_expression_primary */ - { 40, -1 }, /* (50) nonparenthesized_value_expression_primary ::= literal */ - { 40, -1 }, /* (51) nonparenthesized_value_expression_primary ::= column_reference */ - { 41, -1 }, /* (52) literal ::= NK_LITERAL */ - { 38, -1 }, /* (53) value_expression ::= common_value_expression */ - { 44, -1 }, /* (54) common_value_expression ::= numeric_value_expression */ - { 45, -1 }, /* (55) numeric_value_expression ::= numeric_primary */ - { 45, -2 }, /* (56) numeric_value_expression ::= NK_PLUS numeric_primary */ - { 45, -2 }, /* (57) numeric_value_expression ::= NK_MINUS numeric_primary */ - { 45, -3 }, /* (58) numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - { 45, -3 }, /* (59) numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ - { 45, -3 }, /* (60) numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ - { 45, -3 }, /* (61) numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ - { 46, -1 }, /* (62) numeric_primary ::= value_expression_primary */ - { 46, -1 }, /* (63) numeric_primary ::= value_function */ - { 56, -1 }, /* (64) table_primary ::= derived_table */ - { 58, -1 }, /* (65) derived_table ::= table_subquery */ - { 60, -3 }, /* (66) subquery ::= NK_LR query_expression NK_RP */ - { 59, -1 }, /* (67) table_subquery ::= subquery */ + { 45, -2 }, /* (0) cmd ::= SHOW DATABASES */ + { 45, -1 }, /* (1) cmd ::= query_expression */ + { 47, -1 }, /* (2) unsigned_integer ::= NK_INTEGER */ + { 48, -1 }, /* (3) unsigned_approximate_numeric ::= NK_FLOAT */ + { 49, -1 }, /* (4) signed_integer ::= unsigned_integer */ + { 49, -2 }, /* (5) signed_integer ::= NK_PLUS unsigned_integer */ + { 49, -2 }, /* (6) signed_integer ::= NK_MINUS unsigned_integer */ + { 53, -1 }, /* (7) db_name ::= NK_ID */ + { 54, -1 }, /* (8) table_name ::= NK_ID */ + { 55, -1 }, /* (9) column_name ::= NK_ID */ + { 56, -1 }, /* (10) table_alias ::= NK_ID */ + { 61, -1 }, /* (11) column_reference ::= column_name */ + { 61, -3 }, /* (12) column_reference ::= table_name NK_DOT column_name */ + { 63, -1 }, /* (13) boolean_value_expression ::= boolean_primary */ + { 63, -2 }, /* (14) boolean_value_expression ::= NOT boolean_primary */ + { 63, -3 }, /* (15) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 63, -3 }, /* (16) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 64, -1 }, /* (17) boolean_primary ::= predicate */ + { 64, -3 }, /* (18) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 66, -2 }, /* (19) from_clause ::= FROM table_reference_list */ + { 67, -1 }, /* (20) table_reference_list ::= table_reference */ + { 67, -3 }, /* (21) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 68, -1 }, /* (22) table_reference ::= table_primary */ + { 68, -1 }, /* (23) table_reference ::= joined_table */ + { 69, -2 }, /* (24) table_primary ::= table_name correlation_or_recognition_opt */ + { 69, -4 }, /* (25) table_primary ::= db_name NK_DOT table_name correlation_or_recognition_opt */ + { 69, -2 }, /* (26) table_primary ::= subquery correlation_or_recognition_opt */ + { 71, 0 }, /* (27) correlation_or_recognition_opt ::= */ + { 71, -1 }, /* (28) correlation_or_recognition_opt ::= table_alias */ + { 71, -2 }, /* (29) correlation_or_recognition_opt ::= AS table_alias */ + { 72, -3 }, /* (30) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 72, -3 }, /* (31) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 70, -6 }, /* (32) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 73, -1 }, /* (33) join_type ::= INNER */ + { 75, -9 }, /* (34) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 76, 0 }, /* (35) set_quantifier_opt ::= */ + { 76, -1 }, /* (36) set_quantifier_opt ::= DISTINCT */ + { 76, -1 }, /* (37) set_quantifier_opt ::= ALL */ + { 77, -1 }, /* (38) select_list ::= NK_STAR */ + { 77, -1 }, /* (39) select_list ::= select_sublist */ + { 83, -1 }, /* (40) select_sublist ::= select_item */ + { 83, -3 }, /* (41) select_sublist ::= select_sublist NK_COMMA select_item */ + { 84, -1 }, /* (42) select_item ::= value_expression */ + { 84, -2 }, /* (43) select_item ::= value_expression NK_ID */ + { 84, -3 }, /* (44) select_item ::= value_expression AS NK_ID */ + { 84, -3 }, /* (45) select_item ::= table_name NK_DOT NK_STAR */ + { 46, -4 }, /* (46) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 85, -1 }, /* (47) query_expression_body ::= query_primary */ + { 85, -4 }, /* (48) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 89, -1 }, /* (49) query_primary ::= query_specification */ + { 89, -6 }, /* (50) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ + { 86, 0 }, /* (51) order_by_clause_opt ::= */ + { 86, -3 }, /* (52) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 87, 0 }, /* (53) slimit_clause_opt ::= */ + { 87, -2 }, /* (54) slimit_clause_opt ::= SLIMIT signed_integer */ + { 87, -4 }, /* (55) slimit_clause_opt ::= SLIMIT signed_integer SOFFSET signed_integer */ + { 87, -4 }, /* (56) slimit_clause_opt ::= SLIMIT signed_integer NK_COMMA signed_integer */ + { 88, 0 }, /* (57) limit_clause_opt ::= */ + { 88, -2 }, /* (58) limit_clause_opt ::= LIMIT signed_integer */ + { 88, -4 }, /* (59) limit_clause_opt ::= LIMIT signed_integer OFFSET signed_integer */ + { 88, -4 }, /* (60) limit_clause_opt ::= LIMIT signed_integer NK_COMMA signed_integer */ + { 62, -3 }, /* (61) subquery ::= NK_LR query_expression NK_RP */ + { 74, -1 }, /* (62) search_condition ::= boolean_value_expression */ + { 90, -1 }, /* (63) sort_specification_list ::= sort_specification */ + { 90, -3 }, /* (64) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 91, -3 }, /* (65) sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ + { 92, 0 }, /* (66) ordering_specification_opt ::= */ + { 92, -1 }, /* (67) ordering_specification_opt ::= ASC */ + { 92, -1 }, /* (68) ordering_specification_opt ::= DESC */ + { 93, 0 }, /* (69) null_ordering_opt ::= */ + { 93, -2 }, /* (70) null_ordering_opt ::= NULLS FIRST */ + { 93, -2 }, /* (71) null_ordering_opt ::= NULLS LAST */ + { 50, -1 }, /* (72) unsigned_literal ::= unsigned_numeric_literal */ + { 50, -1 }, /* (73) unsigned_literal ::= general_literal */ + { 51, -1 }, /* (74) unsigned_numeric_literal ::= unsigned_integer */ + { 51, -1 }, /* (75) unsigned_numeric_literal ::= unsigned_approximate_numeric */ + { 52, -1 }, /* (76) general_literal ::= NK_STRING */ + { 52, -1 }, /* (77) general_literal ::= NK_BOOL */ + { 52, -1 }, /* (78) general_literal ::= NK_NOW */ + { 57, -1 }, /* (79) unsigned_value_specification ::= unsigned_literal */ + { 57, -1 }, /* (80) unsigned_value_specification ::= NK_QUESTION */ + { 58, -3 }, /* (81) value_expression_primary ::= NK_LP value_expression NK_RP */ + { 58, -1 }, /* (82) value_expression_primary ::= nonparenthesized_value_expression_primary */ + { 60, -1 }, /* (83) nonparenthesized_value_expression_primary ::= unsigned_value_specification */ + { 60, -1 }, /* (84) nonparenthesized_value_expression_primary ::= column_reference */ + { 60, -1 }, /* (85) nonparenthesized_value_expression_primary ::= subquery */ + { 69, -1 }, /* (86) table_primary ::= parenthesized_joined_table */ + { 65, 0 }, /* (87) predicate ::= */ + { 78, 0 }, /* (88) where_clause_opt ::= */ + { 79, 0 }, /* (89) partition_by_clause_opt ::= */ + { 80, 0 }, /* (90) twindow_clause_opt ::= */ + { 81, 0 }, /* (91) group_by_clause_opt ::= */ + { 82, 0 }, /* (92) having_clause_opt ::= */ + { 94, -4 }, /* (93) value_function ::= NK_ID NK_LP value_expression NK_RP */ + { 94, -6 }, /* (94) value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ + { 59, -1 }, /* (95) value_expression ::= common_value_expression */ + { 95, -1 }, /* (96) common_value_expression ::= numeric_value_expression */ + { 96, -1 }, /* (97) numeric_value_expression ::= numeric_primary */ + { 96, -2 }, /* (98) numeric_value_expression ::= NK_PLUS numeric_primary */ + { 96, -2 }, /* (99) numeric_value_expression ::= NK_MINUS numeric_primary */ + { 96, -3 }, /* (100) numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ + { 96, -3 }, /* (101) numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ + { 96, -3 }, /* (102) numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ + { 96, -3 }, /* (103) numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ + { 97, -1 }, /* (104) numeric_primary ::= value_expression_primary */ + { 97, -1 }, /* (105) numeric_primary ::= value_function */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1173,240 +1356,347 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy130; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy168; } break; - case 2: /* column_reference ::= NK_ID */ -{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 2: /* unsigned_integer ::= NK_INTEGER */ +{ yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 3: /* column_reference ::= table_name NK_DOT NK_ID */ - case 14: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==14); -{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 3: /* unsigned_approximate_numeric ::= NK_FLOAT */ +{ yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 4: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause */ -{ PARSER_TRACE; yymsp[-3].minor.yy130 = createSelectStmt(pCxt, yymsp[-2].minor.yy9, yymsp[-1].minor.yy30, yymsp[0].minor.yy130); } + case 4: /* signed_integer ::= unsigned_integer */ +{ yylhsminor.yy168 = yymsp[0].minor.yy168; } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 5: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy9 = false; } + case 5: /* signed_integer ::= NK_PLUS unsigned_integer */ +{ yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } break; - case 6: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy9 = true; } + case 6: /* signed_integer ::= NK_MINUS unsigned_integer */ +{ yymsp[-1].minor.yy168 = addMinusSign(pCxt, yymsp[0].minor.yy168);} break; - case 7: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy9 = false; } + case 7: /* db_name ::= NK_ID */ + case 8: /* table_name ::= NK_ID */ yytestcase(yyruleno==8); + case 9: /* column_name ::= NK_ID */ yytestcase(yyruleno==9); + case 10: /* table_alias ::= NK_ID */ yytestcase(yyruleno==10); +{ PARSER_TRACE; yylhsminor.yy169 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy169 = yylhsminor.yy169; break; - case 8: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy30 = NULL; } + case 11: /* column_reference ::= column_name */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy169); } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 9: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy30 = yymsp[0].minor.yy30; } - yymsp[0].minor.yy30 = yylhsminor.yy30; + case 12: /* column_reference ::= table_name NK_DOT column_name */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy169, &yymsp[0].minor.yy169); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 10: /* select_sublist ::= select_item */ - case 37: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==37); -{ PARSER_TRACE; yylhsminor.yy30 = createNodeList(pCxt, yymsp[0].minor.yy130); } - yymsp[0].minor.yy30 = yylhsminor.yy30; + case 13: /* boolean_value_expression ::= boolean_primary */ + case 17: /* boolean_primary ::= predicate */ yytestcase(yyruleno==17); + case 20: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==20); + case 22: /* table_reference ::= table_primary */ yytestcase(yyruleno==22); + case 23: /* table_reference ::= joined_table */ yytestcase(yyruleno==23); + case 42: /* select_item ::= value_expression */ yytestcase(yyruleno==42); + case 47: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==47); + case 49: /* query_primary ::= query_specification */ yytestcase(yyruleno==49); + case 62: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==62); +{ PARSER_TRACE; yylhsminor.yy168 = yymsp[0].minor.yy168; } + yymsp[0].minor.yy168 = yylhsminor.yy168; break; - case 11: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 38: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==38); -{ PARSER_TRACE; yylhsminor.yy30 = addNodeToList(pCxt, yymsp[-2].minor.yy30, yymsp[0].minor.yy130); } - yymsp[-2].minor.yy30 = yylhsminor.yy30; + case 14: /* boolean_value_expression ::= NOT boolean_primary */ +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy168, NULL); } break; - case 12: /* select_item ::= value_expression */ - case 16: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==16); - case 17: /* table_reference ::= table_factor */ yytestcase(yyruleno==17); - case 18: /* table_factor ::= table_primary */ yytestcase(yyruleno==18); - case 25: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==25); - case 27: /* query_primary ::= query_specification */ yytestcase(yyruleno==27); -{ PARSER_TRACE; yylhsminor.yy130 = yymsp[0].minor.yy130; } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 15: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 13: /* select_item ::= value_expression AS NK_ID */ -{ PARSER_TRACE; yylhsminor.yy130 = setProjectionAlias(pCxt, yymsp[-2].minor.yy130, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 16: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 15: /* from_clause ::= FROM table_reference_list */ -{ PARSER_TRACE; yymsp[-1].minor.yy130 = yymsp[0].minor.yy130; } + case 18: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + case 30: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==30); + case 31: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==31); + case 61: /* subquery ::= NK_LR query_expression NK_RP */ yytestcase(yyruleno==61); +{ PARSER_TRACE; yymsp[-2].minor.yy168 = yymsp[-1].minor.yy168; } break; - case 19: /* table_primary ::= table_name */ -{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy67); } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 19: /* from_clause ::= FROM table_reference_list */ +{ PARSER_TRACE; yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } break; - case 20: /* table_primary ::= db_name NK_DOT table_name */ -{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy67); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 21: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy168, yymsp[0].minor.yy168, NULL); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 21: /* db_name ::= NK_ID */ - case 22: /* table_name ::= NK_ID */ yytestcase(yyruleno==22); -{ PARSER_TRACE; yylhsminor.yy67 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy67 = yylhsminor.yy67; + case 24: /* table_primary ::= table_name correlation_or_recognition_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy169, &yymsp[0].minor.yy169); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; break; - case 23: /* query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ -{ yy_destructor(yypParser,61,&yymsp[-4].minor); + case 25: /* table_primary ::= db_name NK_DOT table_name correlation_or_recognition_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, &yymsp[-3].minor.yy169, &yymsp[-1].minor.yy169, &yymsp[0].minor.yy169); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 26: /* table_primary ::= subquery correlation_or_recognition_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createTempTableNode(pCxt, yymsp[-1].minor.yy168, &yymsp[0].minor.yy169); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; + break; + case 27: /* correlation_or_recognition_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy169 = nil_token; } + break; + case 28: /* correlation_or_recognition_opt ::= table_alias */ +{ PARSER_TRACE; yylhsminor.yy169 = yymsp[0].minor.yy169; } + yymsp[0].minor.yy169 = yylhsminor.yy169; + break; + case 29: /* correlation_or_recognition_opt ::= AS table_alias */ +{ PARSER_TRACE; yymsp[-1].minor.yy169 = yymsp[0].minor.yy169; } + break; + case 32: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, yymsp[-4].minor.yy148, yymsp[-5].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-5].minor.yy168 = yylhsminor.yy168; + break; + case 33: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy148 = JOIN_TYPE_INNER; } + break; + case 34: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - PARSER_TRACE; - addOrderByList(pCxt, yymsp[-3].minor.yy130, yymsp[-2].minor.yy30); - addSlimit(pCxt, yymsp[-3].minor.yy130, yymsp[-1].minor.yy130); - addLimit(pCxt, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); - yymsp[-4].minor.yy130 = yymsp[-3].minor.yy130; - } -} + PARSER_TRACE; + yymsp[-8].minor.yy168 = createSelectStmt(pCxt, yymsp[-7].minor.yy173, yymsp[-6].minor.yy40, yymsp[-5].minor.yy168); + yymsp[-8].minor.yy168 = addWhereClause(pCxt, yymsp[-8].minor.yy168, yymsp[-4].minor.yy168); + yymsp[-8].minor.yy168 = addPartitionByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-3].minor.yy40); + yymsp[-8].minor.yy168 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy168, yymsp[-2].minor.yy168); + yymsp[-8].minor.yy168 = addGroupByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-1].minor.yy40); + yymsp[-8].minor.yy168 = addHavingClause(pCxt, yymsp[-8].minor.yy168, yymsp[0].minor.yy168); + } break; - case 24: /* with_clause_opt ::= */ -{} + case 35: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy173 = false; } break; - case 26: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy130 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); } - yymsp[-3].minor.yy130 = yylhsminor.yy130; + case 36: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy173 = true; } break; - case 28: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy130 = yymsp[-4].minor.yy130;} - yy_destructor(yypParser,63,&yymsp[-3].minor); - yy_destructor(yypParser,65,&yymsp[-2].minor); - yy_destructor(yypParser,64,&yymsp[-1].minor); + case 37: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy173 = false; } break; - case 29: /* order_by_clause_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy30 = NULL; } + case 38: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = NULL; } break; - case 30: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ -{ PARSER_TRACE; yymsp[-2].minor.yy30 = yymsp[0].minor.yy30; } + case 39: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy40 = yymsp[0].minor.yy40; } + yymsp[0].minor.yy40 = yylhsminor.yy40; break; - case 31: /* slimit_clause_opt ::= */ - case 34: /* limit_clause_opt ::= */ yytestcase(yyruleno==34); -{ yymsp[1].minor.yy130 = NULL; } + case 40: /* select_sublist ::= select_item */ + case 63: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==63); +{ PARSER_TRACE; yylhsminor.yy40 = createNodeList(pCxt, yymsp[0].minor.yy168); } + yymsp[0].minor.yy40 = yylhsminor.yy40; break; - case 32: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 35: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==35); -{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 41: /* select_sublist ::= select_sublist NK_COMMA select_item */ + case 64: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==64); +{ PARSER_TRACE; yylhsminor.yy40 = addNodeToList(pCxt, yymsp[-2].minor.yy40, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy40 = yylhsminor.yy40; break; - case 33: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 36: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==36); -{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 43: /* select_item ::= value_expression NK_ID */ +{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, yymsp[-1].minor.yy168, &yymsp[0].minor.yy0); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; break; - case 39: /* sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy130 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy130, yymsp[-1].minor.yy108, yymsp[0].minor.yy68); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 44: /* select_item ::= value_expression AS NK_ID */ +{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, yymsp[-2].minor.yy168, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 40: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy108 = ORDER_ASC; } + case 45: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy169, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; break; - case 41: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_ASC; } + case 46: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ +{ + PARSER_TRACE; + yylhsminor.yy168 = addOrderByClause(pCxt, yymsp[-3].minor.yy168, yymsp[-2].minor.yy40); + yylhsminor.yy168 = addSlimitClause(pCxt, yylhsminor.yy168, yymsp[-1].minor.yy168); + yylhsminor.yy168 = addLimitClause(pCxt, yylhsminor.yy168, yymsp[0].minor.yy168); + } + yymsp[-3].minor.yy168 = yylhsminor.yy168; break; - case 42: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_DESC; } + case 48: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy168 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; break; - case 43: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy68 = NULL_ORDER_DEFAULT; } + case 50: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = yymsp[-4].minor.yy168;} + yy_destructor(yypParser,86,&yymsp[-3].minor); + yy_destructor(yypParser,88,&yymsp[-2].minor); + yy_destructor(yypParser,87,&yymsp[-1].minor); break; - case 44: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_FIRST; } + case 51: /* order_by_clause_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy40 = NULL; } break; - case 45: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_LAST; } + case 52: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ +{ PARSER_TRACE; yymsp[-2].minor.yy40 = yymsp[0].minor.yy40; } break; - case 46: /* value_function ::= NK_ID NK_LP value_expression NK_RP */ - case 48: /* value_expression_primary ::= NK_LP value_expression NK_RP */ yytestcase(yyruleno==48); -{ -} - yy_destructor(yypParser,38,&yymsp[-1].minor); + case 53: /* slimit_clause_opt ::= */ + case 57: /* limit_clause_opt ::= */ yytestcase(yyruleno==57); +{ PARSER_TRACE; yymsp[1].minor.yy168 = NULL; } break; - case 47: /* value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ -{ -} - yy_destructor(yypParser,38,&yymsp[-3].minor); - yy_destructor(yypParser,38,&yymsp[-1].minor); + case 54: /* slimit_clause_opt ::= SLIMIT signed_integer */ + case 58: /* limit_clause_opt ::= LIMIT signed_integer */ yytestcase(yyruleno==58); +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLimitNode(pCxt, yymsp[0].minor.yy168, 0); } break; - case 49: /* value_expression_primary ::= nonparenthesized_value_expression_primary */ -{ yy_destructor(yypParser,40,&yymsp[0].minor); + case 55: /* slimit_clause_opt ::= SLIMIT signed_integer SOFFSET signed_integer */ + case 59: /* limit_clause_opt ::= LIMIT signed_integer OFFSET signed_integer */ yytestcase(yyruleno==59); +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + break; + case 56: /* slimit_clause_opt ::= SLIMIT signed_integer NK_COMMA signed_integer */ + case 60: /* limit_clause_opt ::= LIMIT signed_integer NK_COMMA signed_integer */ yytestcase(yyruleno==60); +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, yymsp[0].minor.yy168, yymsp[-2].minor.yy168); } + break; + case 65: /* sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy168, yymsp[-1].minor.yy106, yymsp[0].minor.yy193); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 66: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy106 = ORDER_ASC; } + break; + case 67: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy106 = ORDER_ASC; } + break; + case 68: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy106 = ORDER_DESC; } + break; + case 69: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy193 = NULL_ORDER_DEFAULT; } + break; + case 70: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy193 = NULL_ORDER_FIRST; } + break; + case 71: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy193 = NULL_ORDER_LAST; } + break; + case 72: /* unsigned_literal ::= unsigned_numeric_literal */ +{ yy_destructor(yypParser,51,&yymsp[0].minor); { } } break; - case 50: /* nonparenthesized_value_expression_primary ::= literal */ -{ yy_destructor(yypParser,41,&yymsp[0].minor); + case 73: /* unsigned_literal ::= general_literal */ +{ yy_destructor(yypParser,52,&yymsp[0].minor); { } } break; - case 51: /* nonparenthesized_value_expression_primary ::= column_reference */ -{ yy_destructor(yypParser,42,&yymsp[0].minor); + case 74: /* unsigned_numeric_literal ::= unsigned_integer */ +{ yy_destructor(yypParser,47,&yymsp[0].minor); { } } break; - case 53: /* value_expression ::= common_value_expression */ -{ yy_destructor(yypParser,44,&yymsp[0].minor); + case 75: /* unsigned_numeric_literal ::= unsigned_approximate_numeric */ +{ yy_destructor(yypParser,48,&yymsp[0].minor); { } } break; - case 54: /* common_value_expression ::= numeric_value_expression */ -{ yy_destructor(yypParser,45,&yymsp[0].minor); + case 79: /* unsigned_value_specification ::= unsigned_literal */ +{ yy_destructor(yypParser,50,&yymsp[0].minor); { } } break; - case 55: /* numeric_value_expression ::= numeric_primary */ -{ yy_destructor(yypParser,46,&yymsp[0].minor); + case 81: /* value_expression_primary ::= NK_LP value_expression NK_RP */ + case 93: /* value_function ::= NK_ID NK_LP value_expression NK_RP */ yytestcase(yyruleno==93); { } -} + yy_destructor(yypParser,59,&yymsp[-1].minor); break; - case 56: /* numeric_value_expression ::= NK_PLUS numeric_primary */ - case 57: /* numeric_value_expression ::= NK_MINUS numeric_primary */ yytestcase(yyruleno==57); -{ -} - yy_destructor(yypParser,46,&yymsp[0].minor); - break; - case 58: /* numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - case 59: /* numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ yytestcase(yyruleno==59); - case 60: /* numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ yytestcase(yyruleno==60); - case 61: /* numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ yytestcase(yyruleno==61); -{ yy_destructor(yypParser,45,&yymsp[-2].minor); -{ -} - yy_destructor(yypParser,45,&yymsp[0].minor); -} - break; - case 62: /* numeric_primary ::= value_expression_primary */ -{ yy_destructor(yypParser,39,&yymsp[0].minor); -{ -} -} - break; - case 63: /* numeric_primary ::= value_function */ -{ yy_destructor(yypParser,37,&yymsp[0].minor); -{ -} -} - break; - case 64: /* table_primary ::= derived_table */ -{ yy_destructor(yypParser,58,&yymsp[0].minor); -{ -} -} - break; - case 65: /* derived_table ::= table_subquery */ -{ yy_destructor(yypParser,59,&yymsp[0].minor); -{ -} -} - break; - case 66: /* subquery ::= NK_LR query_expression NK_RP */ -{ -} - yy_destructor(yypParser,36,&yymsp[-1].minor); - break; - case 67: /* table_subquery ::= subquery */ + case 82: /* value_expression_primary ::= nonparenthesized_value_expression_primary */ { yy_destructor(yypParser,60,&yymsp[0].minor); { } +} + break; + case 83: /* nonparenthesized_value_expression_primary ::= unsigned_value_specification */ +{ yy_destructor(yypParser,57,&yymsp[0].minor); +{ +} +} + break; + case 84: /* nonparenthesized_value_expression_primary ::= column_reference */ +{ yy_destructor(yypParser,61,&yymsp[0].minor); +{ +} +} + break; + case 85: /* nonparenthesized_value_expression_primary ::= subquery */ +{ yy_destructor(yypParser,62,&yymsp[0].minor); +{ +} +} + break; + case 86: /* table_primary ::= parenthesized_joined_table */ +{ yy_destructor(yypParser,72,&yymsp[0].minor); +{ +} +} + break; + case 94: /* value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ +{ +} + yy_destructor(yypParser,59,&yymsp[-3].minor); + yy_destructor(yypParser,59,&yymsp[-1].minor); + break; + case 95: /* value_expression ::= common_value_expression */ +{ yy_destructor(yypParser,95,&yymsp[0].minor); +{ +} +} + break; + case 96: /* common_value_expression ::= numeric_value_expression */ +{ yy_destructor(yypParser,96,&yymsp[0].minor); +{ +} +} + break; + case 97: /* numeric_value_expression ::= numeric_primary */ +{ yy_destructor(yypParser,97,&yymsp[0].minor); +{ +} +} + break; + case 98: /* numeric_value_expression ::= NK_PLUS numeric_primary */ + case 99: /* numeric_value_expression ::= NK_MINUS numeric_primary */ yytestcase(yyruleno==99); +{ +} + yy_destructor(yypParser,97,&yymsp[0].minor); + break; + case 100: /* numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ + case 101: /* numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ yytestcase(yyruleno==101); + case 102: /* numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ yytestcase(yyruleno==102); + case 103: /* numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ yytestcase(yyruleno==103); +{ yy_destructor(yypParser,96,&yymsp[-2].minor); +{ +} + yy_destructor(yypParser,96,&yymsp[0].minor); +} + break; + case 104: /* numeric_primary ::= value_expression_primary */ +{ yy_destructor(yypParser,58,&yymsp[0].minor); +{ +} +} + break; + case 105: /* numeric_primary ::= value_function */ +{ yy_destructor(yypParser,94,&yymsp[0].minor); +{ +} } break; default: - /* (52) literal ::= NK_LITERAL */ yytestcase(yyruleno==52); + /* (76) general_literal ::= NK_STRING */ yytestcase(yyruleno==76); + /* (77) general_literal ::= NK_BOOL */ yytestcase(yyruleno==77); + /* (78) general_literal ::= NK_NOW */ yytestcase(yyruleno==78); + /* (80) unsigned_value_specification ::= NK_QUESTION */ yytestcase(yyruleno==80); + /* (87) predicate ::= */ yytestcase(yyruleno==87); + /* (88) where_clause_opt ::= */ yytestcase(yyruleno==88); + /* (89) partition_by_clause_opt ::= */ yytestcase(yyruleno==89); + /* (90) twindow_clause_opt ::= */ yytestcase(yyruleno==90); + /* (91) group_by_clause_opt ::= */ yytestcase(yyruleno==91); + /* (92) having_clause_opt ::= */ yytestcase(yyruleno==92); break; /********** End reduce actions ************************************************/ }; diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 8902da9274..09675d0917 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -80,7 +80,8 @@ uint32_t getToken(const char* z, uint32_t* tokenId) { } int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { - SAstCreateContext cxt = { .pQueryCxt = pParseCxt, .valid = true, .pRootNode = NULL }; + SAstCreateContext cxt; + createAstCreateContext(pParseCxt, &cxt); void *pParser = NewParseAlloc(malloc); int32_t i = 0; while (1) { @@ -93,7 +94,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); t0.z = (char *)(cxt.pQueryCxt->pSql + i); - printf("token %p : %d %d [%s]\n", &t0, t0.type, t0.n, t0.z); + printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); i += t0.n; switch (t0.type) { @@ -130,7 +131,9 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { } abort_parse: + printf("doParse completed.\n"); NewParseFree(pParser, free); + destroyAstCreateContext(&cxt); pQuery->pRoot = cxt.pRootNode; return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 5223c9f02a..395b61ea20 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -38,11 +38,11 @@ protected: } - int32_t run() { + bool run(int32_t expectCode = TSDB_CODE_SUCCESS) { int32_t code = doParse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return code; + return (code == expectCode); } cout << nodeType(query_.pRoot) << endl; if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { @@ -76,7 +76,7 @@ protected: // return code; // } // cout << "node tree:\n" << pStr << endl; - return TSDB_CODE_SUCCESS; + return (code == expectCode); } private: @@ -136,8 +136,24 @@ TEST_F(NewParserTest, selectStar) { setDatabase("root", "test"); bind("SELECT * FROM t1"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + ASSERT_TRUE(run()); bind("SELECT * FROM test.t1"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + ASSERT_TRUE(run()); +} + +TEST_F(NewParserTest, syntaxError) { + setDatabase("root", "test"); + + bind("SELECTT * FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *, * FROM test.t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT * FROM test.t1 t WHER"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); } diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index 535bd8e171..a0384d1674 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -15,17 +15,13 @@ #include "nodes.h" #include "nodesShowStmts.h" - -bool nodesIsTimeorderQuery(const SNode* pQuery) { - -} - -bool nodesIsTimelineQuery(const SNode* pQuery) { - -} +#include "taoserror.h" static SNode* makeNode(ENodeType type, size_t size) { SNode* p = calloc(1, size); + if (NULL == p) { + return NULL; + } setNodeType(p, type); return p; } @@ -78,6 +74,37 @@ void nodesDestroyNode(SNode* pNode) { } -void nodesDestroyNodeList(SNodeList* pList) { +SNodeList* nodesMakeList() { + SNodeList* p = calloc(1, sizeof(SNodeList)); + if (NULL == p) { + return NULL; + } + return p; +} + +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { + if (NULL == pList || NULL == pNode) { + return NULL; + } + SListCell* p = calloc(1, sizeof(SListCell)); + if (NULL == p) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pList; + } + p->pNode = pNode; + pList->pTail->pNext = p; + pList->pTail = p; + return pList; +} + +void nodesDestroyList(SNodeList* pList) { } + +bool nodesIsTimeorderQuery(const SNode* pQuery) { + +} + +bool nodesIsTimelineQuery(const SNode* pQuery) { + +} \ No newline at end of file From 6da9c663ff78faf06ed53578092c5be291e0dec9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Jan 2022 15:54:49 +0800 Subject: [PATCH 03/42] [td-11818] update log, add sql in the log file. Remove a global configure parameter. --- include/common/tglobal.h | 1 - include/common/tmsg.h | 16 +++- include/libs/scheduler/scheduler.h | 4 +- source/client/src/clientImpl.c | 13 ++-- source/common/src/tglobal.c | 11 --- source/libs/qcom/src/queryUtil.c | 2 +- source/libs/qworker/inc/qworkerInt.h | 6 +- source/libs/qworker/src/qworker.c | 3 +- source/libs/qworker/src/qworkerMsg.c | 15 ++-- source/libs/qworker/test/qworkerTests.cpp | 3 +- source/libs/scheduler/inc/schedulerInt.h | 7 +- source/libs/scheduler/src/scheduler.c | 76 +++++++------------ source/libs/scheduler/test/schedulerTests.cpp | 6 +- 13 files changed, 75 insertions(+), 88 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1b96cbdc78..4bc8c9fb86 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -64,7 +64,6 @@ extern int8_t tsKeepOriginalColumnName; extern int8_t tsDeadLockKillQuery; // client -extern int32_t tsMaxSQLStringLen; extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; extern int8_t tsTscEnableRecordSql; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 765022ebaa..8b4dc67c17 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -864,13 +864,21 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupReq; +/** + * The layout of the query message payload is as following: + * +--------------------+---------------------------------+ + * |Sql statement | Physical plan | + * |(denoted by sqlLen) |(In JSON, denoted by contentLen) | + * +--------------------+---------------------------------+ + */ typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; int8_t taskType; - uint32_t contentLen; + uint32_t sqlLen; // the query sql, + uint32_t phyLen; char msg[]; } SSubQueryMsg; @@ -1534,7 +1542,8 @@ static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* tlen += taosEncodeFixedU64(buf, pMsg->sId); tlen += taosEncodeFixedU64(buf, pMsg->queryId); tlen += taosEncodeFixedU64(buf, pMsg->taskId); - tlen += taosEncodeFixedU32(buf, pMsg->contentLen); + tlen += taosEncodeFixedU32(buf, pMsg->sqlLen); + tlen += taosEncodeFixedU32(buf, pMsg->phyLen); //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } @@ -1543,7 +1552,8 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { buf = taosDecodeFixedU64(buf, &pMsg->sId); buf = taosDecodeFixedU64(buf, &pMsg->queryId); buf = taosDecodeFixedU64(buf, &pMsg->taskId); - buf = taosDecodeFixedU32(buf, &pMsg->contentLen); + buf = taosDecodeFixedU32(buf, &pMsg->sqlLen); + buf = taosDecodeFixedU32(buf, &pMsg->phyLen); //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); return buf; } diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 5a0da46490..e856adaf31 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -80,7 +80,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob); /** * Fetch query result from the remote query executor diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 247b543dea..63da5862d3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -239,9 +239,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { + void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); + int32_t code = schedulerExecJob(pTransporter, NULL, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { @@ -255,7 +256,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) return pRequest->code; } - return schedulerAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); + return schedulerAsyncExecJob(pTransporter, pNodeList, pDag, pRequest->sqlstr, &pRequest->body.pQueryJob); } @@ -532,8 +533,8 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, goto _return; } - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; goto _return; } @@ -755,8 +756,8 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; - if (sqlLen > (size_t) tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + if (sqlLen > (size_t) TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a353c018b7..afba1190ba 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -78,7 +78,6 @@ int32_t tsCompressColData = -1; int32_t tsCompatibleModel = 1; // client -int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; @@ -594,16 +593,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "maxSQLLength"; - cfg.ptr = &tsMaxSQLStringLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MAX_SQL_LEN; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cfe2ed96ad..9f678b4528 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -128,7 +128,7 @@ int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransp if (NULL == pMsg) { qError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return -1; + return terrno; } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index a19a02069b..de2940846c 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -182,9 +182,9 @@ typedef struct SQWorkerMgmt { #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) #define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9b9b6a73ca..8705bc23b9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -479,7 +479,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { if (NULL == pRes) { QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); - dsEndPut(sinkHandle, useconds); if (TASK_TYPE_TEMP == ctx->taskType) { @@ -493,6 +492,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { break; } + ASSERT(pRes->info.rows > 0); + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); if (code) { diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 11783335b3..368a590918 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -282,18 +282,21 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); + msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->contentLen = ntohl(msg->contentLen); - + msg->taskId = be64toh(msg->taskId); + msg->phyLen = ntohl(msg->phyLen); + msg->sqlLen = ntohl(msg->sqlLen); + uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - SQWMsg qwMsg = {.node = node, .msg = msg->msg, .msgLen = msg->contentLen, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connection = pMsg}; - QW_SCH_TASK_DLOG("processQuery start, node:%p", node); + char* sql = strndup(msg->msg, msg->sqlLen); + QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql); + tfree(sql); QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 4c81123bf2..38890150ee 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -113,7 +113,8 @@ void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { qwtqueryMsg.queryId = htobe64(atomic_add_fetch_64(&qwtTestQueryId, 1)); qwtqueryMsg.sId = htobe64(1); qwtqueryMsg.taskId = htobe64(1); - qwtqueryMsg.contentLen = htonl(100); + qwtqueryMsg.phyLen = htonl(100); + qwtqueryMsg.sqlLen = 0; queryRpc->msgType = TDMT_VND_QUERY; queryRpc->pCont = &qwtqueryMsg; queryRpc->contLen = sizeof(SSubQueryMsg) + 100; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 2ace1c66f5..f5e3ef9963 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -128,6 +128,7 @@ typedef struct SSchJob { int32_t errCode; void *res; //TODO free it or not int32_t resNumOfRows; + const char *sql; SQueryProfileSummary summary; } SSchJob; @@ -150,11 +151,11 @@ typedef struct SSchJob { #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 621ed68bae..25137beed9 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -345,7 +345,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { } pLevel = taosArrayGet(pJob->levels, i); - pLevel->level = i; plans = taosArrayGetP(pDag->pSubplans, i); @@ -375,7 +374,7 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - + SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); void *p = taosArrayPush(pLevel->subTasks, &task); @@ -388,8 +387,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - - SCH_TASK_DLOG("task initialized, level:%d", pLevel->level); } SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); @@ -716,8 +713,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; SSchTask *pErrTask = pTask; - SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); - code = schMoveTaskToSuccList(pJob, pTask, &moved); if (code && moved) { SCH_ERR_RET(code); @@ -739,7 +734,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); - return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum); @@ -1031,22 +1025,18 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t pMsgSendInfo->fp = fp; int64_t transporterId = 0; - code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo); if (code) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); SCH_ERR_JRET(code); } - qDebug("QID:%"PRIx64 ",TID:%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); - + qDebug("QID:0x%"PRIx64 ",TID:0x%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: tfree(param); tfree(pMsgSendInfo); - SCH_RET(code); } @@ -1057,7 +1047,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, bool isCandidateAddr = false; if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); - isCandidateAddr = true; } @@ -1078,7 +1067,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY: { - msgSize = sizeof(SSubQueryMsg) + pTask->msgLen; + uint32_t len = strlen(pJob->sql); + + msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); @@ -1086,15 +1077,16 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SSubQueryMsg *pMsg = msg; - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->taskType = TASK_TYPE_TEMP; - pMsg->contentLen = htonl(pTask->msgLen); - memcpy(pMsg->msg, pTask->msg, pTask->msgLen); + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(pJob->queryId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskType = TASK_TYPE_TEMP; + pMsg->phyLen = htonl(pTask->msgLen); + pMsg->sqlLen = htonl(len); + + memcpy(pMsg->msg, pJob->sql, len); + memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); break; } @@ -1205,7 +1197,7 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); } else { - SCH_TASK_DLOG(" ===physical plan=== len:%d, %s", pTask->msgLen, pTask->msg); + SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } @@ -1214,7 +1206,6 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); } @@ -1281,7 +1272,7 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, const char* sql, bool syncSchedule) { qDebug("QID:0x%"PRIx64" job started", pDag->queryId); if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { @@ -1297,6 +1288,7 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; + pJob->sql = sql; if (pNodeList != NULL) { pJob->nodeList = taosArrayDup(pNodeList); @@ -1336,7 +1328,6 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru } pJob->status = JOB_TASK_STATUS_NOT_START; - SCH_ERR_JRET(schLaunchJob(pJob)); *(SSchJob **)job = pJob; @@ -1347,15 +1338,12 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru } SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); - return TSDB_CODE_SUCCESS; _return: *(SSchJob **)job = NULL; - schedulerFreeJob(pJob); - SCH_RET(code); } @@ -1399,29 +1387,24 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = NULL; - - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, true)); - - job = *pJob; - - pRes->code = atomic_load_32(&job->errCode); - pRes->numOfRows = job->resNumOfRows; + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + pRes->code = atomic_load_32(&(*pJob)->errCode); + pRes->numOfRows = (*pJob)->resNumOfRows; return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); return TSDB_CODE_SUCCESS; } @@ -1456,7 +1439,6 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { for (int32_t i = 0; i < taskNum; ++i) { SSubplan *plan = taosArrayGetP(plans, i); - tInfo.addr = plan->execNode; code = qSubPlanToString(plan, &msg, &msgLen); @@ -1472,16 +1454,16 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { } SSubQueryMsg* pMsg = calloc(1, msgSize); - /*SSubQueryMsg *pMsg = (SSubQueryMsg*) msg;*/ memcpy(pMsg->msg, msg, msgLen); pMsg->header.vgId = tInfo.addr.nodeId; - pMsg->sId = schMgmt.sId; - pMsg->queryId = plan->id.queryId; - pMsg->taskId = schGenUUID(); + pMsg->sId = schMgmt.sId; + pMsg->queryId = plan->id.queryId; + pMsg->taskId = schGenUUID(); pMsg->taskType = TASK_TYPE_PERSISTENT; - pMsg->contentLen = msgLen; + pMsg->phyLen = msgLen; + pMsg->sqlLen = 0; /*memcpy(pMsg->msg, ((SSubQueryMsg*)msg)->msg, msgLen);*/ tInfo.msg = pMsg; @@ -1514,7 +1496,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgSize = src->msg->contentLen + sizeof(*src->msg); + int32_t msgSize = src->msg->phyLen + sizeof(*src->msg); STaskInfo info = {0}; info.addr = src->addr; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6e44d99c15..2c75683ecb 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -378,7 +378,7 @@ void* schtRunJobThread(void *aa) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); assert(code == 0); execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); @@ -539,7 +539,7 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &pJob); ASSERT_EQ(code, 0); SSchJob *job = (SSchJob *)pJob; @@ -649,7 +649,7 @@ TEST(insertTest, normalCase) { pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, "insert into tb values(now,1)", &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); From 284fde6001baa7a5583ce5b0d99753b427efc88d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Jan 2022 16:03:26 +0800 Subject: [PATCH 04/42] [td-11818] fix compiler error. --- tools/shell/src/shellEngine.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index b785ed2707..4186f0dae1 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -966,7 +966,7 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char *cmd = calloc(1, tsMaxSQLStringLen + 1); + char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; size_t line_len = 0; @@ -998,7 +998,7 @@ void source_file(TAOS *con, char *fptr) { } while ((read_len = tgetline(&line, &line_len, f)) != -1) { - if (read_len >= tsMaxSQLStringLen) continue; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -1015,7 +1015,7 @@ void source_file(TAOS *con, char *fptr) { memcpy(cmd + cmd_len, line, read_len); printf("%s%s\n", PROMPT_HEADER, cmd); shellRunCommand(con, cmd); - memset(cmd, 0, tsMaxSQLStringLen); + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } From 0712f87de051303fc989213cc94fab0d475f33e1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 27 Jan 2022 16:35:15 +0800 Subject: [PATCH 05/42] fix query crash --- include/common/common.h | 12 ++ include/common/tmsg.h | 7 + source/client/src/clientImpl.c | 156 +++++++++++++++------ source/client/test/clientTests.cpp | 6 +- source/dnode/mnode/impl/inc/mndDef.h | 10 +- source/dnode/mnode/impl/src/mndSubscribe.c | 114 +++++++++++---- source/dnode/vnode/src/tq/tq.c | 17 ++- 7 files changed, 244 insertions(+), 78 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index 0299a29eb4..29ef6d953f 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -126,6 +126,12 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); + if (pRsp->numOfTopics == 0) return tlen; tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); if (pRsp->pBlockData) { sz = taosArrayGetSize(pRsp->pBlockData); @@ -141,6 +147,12 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); + if (pRsp->numOfTopics == 0) return buf; pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); if (pRsp->schemas == NULL) return NULL; buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 289f2143ab..21d5ec021f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1655,6 +1655,10 @@ typedef struct SMqTopicBlk { typedef struct SMqConsumeRsp { int64_t consumerId; SSchemaWrapper* schemas; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; int32_t numOfTopics; SArray* pBlockData; //SArray } SMqConsumeRsp; @@ -1688,6 +1692,7 @@ typedef struct SMqSubTopicEp { typedef struct SMqCMGetSubEpRsp { int64_t consumerId; + int64_t epoch; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray } SMqCMGetSubEpRsp; @@ -1736,6 +1741,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeFixedI64(buf, pRsp->epoch); tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -1748,6 +1754,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeFixedI64(buf, &pRsp->epoch); buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index afac8acc7a..56f89e30c4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -326,13 +326,17 @@ int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { struct tmq_t { char groupId[256]; char clientId[256]; + SRWLatch lock; int64_t consumerId; + int64_t epoch; int64_t status; tsem_t rspSem; STscObj* pTscObj; tmq_commit_cb* commit_cb; int32_t nextTopicIdx; SArray* clientTopics; //SArray + //stat + int64_t pollCnt; }; tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { @@ -342,6 +346,9 @@ tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t err } pTmq->pTscObj = (STscObj*)conn; pTmq->status = 0; + pTmq->pollCnt = 0; + pTmq->epoch = 0; + taosInitRWLatch(&pTmq->lock); strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); pTmq->commit_cb = conf->commit_cb; @@ -621,34 +628,61 @@ struct tmq_message_t { }; int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { + SMqClientVg* pVg = (SMqClientVg*)param; SMqConsumeRsp rsp; tDecodeSMqConsumeRsp(pMsg->pData, &rsp); + if (rsp.numOfTopics == 0) { + /*printf("no data\n");*/ + return 0; + } int32_t colNum = rsp.schemas->nCols; + pVg->currentOffset = rsp.rspOffset; + /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ + /*printf("-----msg begin----\n");*/ + printf("|"); for (int32_t i = 0; i < colNum; i++) { - printf("| %s |", rsp.schemas->pSchema[i].name); + printf(" %15s |", rsp.schemas->pSchema[i].name); } printf("\n"); + printf("=====================================\n"); int32_t sz = taosArrayGetSize(rsp.pBlockData); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < colNum; j++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, j); - for (int32_t k = 0; k < rows; k++) { - void* var = POINTER_SHIFT(pColInfoData->pData, k * pColInfoData->info.bytes); - if (j == 0) printf(" %ld ", *(int64_t*)var); - if (j == 1) printf(" %d ", *(int32_t*)var); + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch(pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + printf(" %15lu |", *(uint64_t*)var); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t*)var); + break; + } } + printf("\n"); } - /*pDataBlock->*/ } + /*printf("\n-----msg end------\n");*/ return 0; } +typedef struct SMqAskEpCbParam { + tmq_t* tmq; + int32_t wait; +} SMqAskEpCbParam; + int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { - tmq_t* tmq = (tmq_t*)param; + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + tmq_t* tmq = pParam->tmq; if (code != 0) { - tsem_post(&tmq->rspSem); + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } return 0; } tscDebug("tmq ask ep cb called"); @@ -657,36 +691,47 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); int32_t sz = taosArrayGetSize(rsp.topics); // TODO: lock - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); - topic.topicName = strdup(pTopicEp->topic); - int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgSz; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - SMqClientVg clientVg = { - .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; + if (rsp.epoch != tmq->epoch) { + /*printf("rsp epoch %ld", rsp.epoch);*/ + /*printf("tmq epoch %ld", tmq->epoch);*/ + //TODO + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); + topic.topicName = strdup(pTopicEp->topic); + int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgSz; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + SMqClientVg clientVg = { + .pollCnt = 0, + .committedOffset = -1, + .currentOffset = -1, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet + }; + taosArrayPush(topic.vgs, &clientVg); + set = true; + } + taosArrayPush(tmq->clientTopics, &topic); } - taosArrayPush(tmq->clientTopics, &topic); + tmq->epoch = rsp.epoch; + } + if (set) { + atomic_store_64(&tmq->status, 1); } - if(set) tmq->status = 1; // unlock - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + free(pParam); return 0; } -tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { - - if (taosArrayGetSize(tmq->clientTopics) == 0 || tmq->status == 0) { +int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { @@ -702,9 +747,17 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + free(buf); + goto END; + } + pParam->tmq = tmq; + pParam->wait = wait; + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; - sendInfo->param = tmq; + sendInfo->param = pParam; sendInfo->fp = tmq_ask_ep_cb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -712,11 +765,20 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(&tmq->rspSem); - } +END: + if (wait) tsem_wait(&tmq->rspSem); + return 0; +} + +tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { + int64_t status = atomic_load_64(&tmq->status); + tmqAsyncAskEp(tmq, status == 0 || taosArrayGetSize(tmq->clientTopics)); + + if (blocking_time < 0) blocking_time = 500; if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); + usleep(blocking_time * 1000); return NULL; } @@ -730,10 +792,15 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); strcpy(pReq->topic, pTopic->topicName); - int32_t nextVgIdx = pTopic->nextVgIdx; - pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx); - pReq->offset = pVg->currentOffset; + int32_t vgSz = taosArrayGetSize(pTopic->vgs); + if (vgSz == 0) { + free(pReq); + usleep(blocking_time * 1000); + return NULL; + } + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % vgSz); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + pReq->offset = pVg->currentOffset+1; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -743,13 +810,16 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; - /*sendInfo->param = &tmq_message;*/ + sendInfo->param = pVg; sendInfo->fp = tmq_poll_cb_inner; + /*printf("req offset: %ld\n", pReq->offset);*/ + int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + tmq->pollCnt++; - tsem_wait(&pRequest->body.rspSem); + usleep(blocking_time * 1000); return tmq_message; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b73079741c..b2a6724e72 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -570,7 +570,6 @@ TEST(testCase, create_topic_Test) { //taos_close(pConn); //} -#if 0 TEST(testCase, tmq_subscribe_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -590,12 +589,11 @@ TEST(testCase, tmq_subscribe_Test) { tmq_subscribe(tmq, topic_list); while (1) { - tmq_message_t* msg = tmq_consume_poll(tmq, 0); - printf("get msg\n"); + tmq_message_t* msg = tmq_consume_poll(tmq, 1000); + //printf("get msg\n"); //if (msg == NULL) break; } } -#endif TEST(testCase, tmq_consume_Test) { } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5ec9173fc8..891f731a42 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -658,12 +658,17 @@ typedef struct SMqConsumerObj { SRWLatch lock; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray - // SHashObj *topicHash; //SHashObj + int64_t epoch; + // stat + int64_t pollCnt; } SMqConsumerObj; static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumer->connId); + tlen += taosEncodeFixedI64(buf, pConsumer->epoch); + tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt); tlen += taosEncodeString(buf, pConsumer->cgroup); int32_t sz = taosArrayGetSize(pConsumer->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -676,6 +681,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerO static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) { buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumer->connId); + buf = taosDecodeFixedI64(buf, &pConsumer->epoch); + buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt); buf = taosDecodeStringTo(buf, pConsumer->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 87f66297a4..2c445a59e3 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -30,6 +30,8 @@ #define MND_SUBSCRIBE_VER_NUMBER 1 #define MND_SUBSCRIBE_RESERVE_SIZE 64 +#define MND_SUBSCRIBE_REBALANCE_MS 5000 + static char *mndMakeSubscribeKey(char *cgroup, char *topicName); static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); @@ -69,6 +71,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqCMGetSubEpRsp rsp; int64_t consumerId = be64toh(pReq->consumerId); + int64_t currentTs = taosGetTimestampMs(); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); if (pConsumer == NULL) { @@ -79,6 +82,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(rsp.cgroup, pReq->cgroup); rsp.consumerId = consumerId; + rsp.epoch = pConsumer->epoch; SArray *pTopics = pConsumer->topics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -88,21 +92,43 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, pConsumerTopic->name); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, pConsumerTopic->name); + ASSERT(pSub); + bool found = 0; + bool changed = 0; + for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + if (*(int64_t*)taosArrayGet(pSub->availConsumer, j) == consumerId) { + found = 1; + break; + } + } + if (found == 0) { + taosArrayPush(pSub->availConsumer, &consumerId); + } + SSdbRaw* pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWriteNotFree(pMnode->pSdb, pRaw); + int32_t assignedSz = taosArrayGetSize(pSub->assigned); topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); for (int32_t j = 0; j < assignedSz; j++) { SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, j); if (pCEp->consumerId == consumerId) { + pCEp->lastConsumerHbTs = currentTs; SMqSubVgEp vgEp = { .epSet = pCEp->epSet, .vgId = pCEp->vgId }; taosArrayPush(topicEp.vgs, &vgEp); + changed = 1; } } if (taosArrayGetSize(topicEp.vgs) != 0) { taosArrayPush(rsp.topics, &topicEp); } + if (changed || found) { + + } + mndReleaseSubscribe(pMnode, pSub); } int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); @@ -124,9 +150,9 @@ static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { i++; } key[i] = 0; - *topic = strdup(key); + *cgroup = strdup(key); key[i] = ':'; - *cgroup = strdup(&key[i + 1]); + *topic = strdup(&key[i + 1]); return 0; } @@ -135,9 +161,37 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SSdb *pSdb = pMnode->pSdb; SMqSubscribeObj *pSub = NULL; void *pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); - int sz; + int64_t currentTs = taosGetTimestampMs(); + int32_t sz; while (pIter != NULL) { - if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) { + for (int i = 0; i < taosArrayGetSize(pSub->assigned); i++) { + SMqConsumerEp* pCEp = taosArrayGet(pSub->assigned, i); + int64_t consumerId = pCEp->consumerId; + if(pCEp->lastConsumerHbTs != -1 && currentTs - pCEp->lastConsumerHbTs > MND_SUBSCRIBE_REBALANCE_MS) { + // put consumer into lostConsumer + taosArrayPush(pSub->lostConsumer, pCEp); + // put vg into unassgined + taosArrayPush(pSub->unassignedVg, pCEp); + // remove from assigned + // TODO: swap with last one, reduce size and reset i + taosArrayRemove(pSub->assigned, i); + // remove from available consumer + for (int j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + if (*(int64_t*)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) { + taosArrayRemove(pSub->availConsumer, j); + break; + } + // TODO: acquire consumer, set status to unavail + } + SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId); + pConsumer->epoch++; + SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWriteNotFree(pMnode->pSdb, pRaw); + mndReleaseConsumer(pMnode, pConsumer); + } + } + if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0 && taosArrayGetSize(pSub->availConsumer) > 0) { char *topic = NULL; char *cgroup = NULL; mndSplitSubscribeKey(pSub->key, &topic, &cgroup); @@ -146,7 +200,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // create trans STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx); SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); pCEp->consumerId = consumerId; @@ -155,49 +209,49 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // build msg - SMqSetCVgReq *pReq = malloc(sizeof(SMqSetCVgReq)); - if (pReq == NULL) { + SMqSetCVgReq req = {0}; + strcpy(req.cgroup, cgroup); + strcpy(req.topicName, topic); + req.sql = pTopic->sql; + req.logicalPlan = pTopic->logicalPlan; + req.physicalPlan = pTopic->physicalPlan; + req.qmsg = pCEp->qmsg; + int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + void *buf = malloc(tlen); + if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - strcpy(pReq->cgroup, cgroup); - strcpy(pReq->topicName, topic); - pReq->sql = strdup(pTopic->sql); - pReq->logicalPlan = strdup(pTopic->logicalPlan); - pReq->physicalPlan = strdup(pTopic->physicalPlan); - pReq->qmsg = strdup(pCEp->qmsg); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); - void *reqStr = malloc(tlen); - if (reqStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - void *abuf = reqStr; - tEncodeSMqSetCVgReq(&abuf, pReq); + SMsgHead *pMsgHead = (SMsgHead *)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); + pMsgHead->vgId = htonl(pCEp->vgId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSMqSetCVgReq(&abuf, &req); // persist msg STransAction action = {0}; action.epSet = pCEp->epSet; - action.pCont = reqStr; + action.pCont = buf; action.contLen = tlen; action.msgType = TDMT_VND_MQ_SET_CONN; mndTransAppendRedoAction(pTrans, &action); // persist raw SSdbRaw *pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); - free(pReq); tfree(topic); tfree(cgroup); } if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); } - /*mndReleaseTopic(pMnode, pTopic);*/ + mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); } - pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); } return 0; } @@ -434,10 +488,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pConsumer->epoch = 1; pConsumer->consumerId = consumerId; strcpy(pConsumer->cgroup, consumerGroup); taosInitRWLatch(&pConsumer->lock); } else { + pConsumer->epoch++; oldSub = pConsumer->topics; } pConsumer->topics = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); @@ -541,6 +597,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); + bool create = false; if (pSub == NULL) { mDebug("create new subscription, group: %s, topic %s", consumerGroup, newTopicName); pSub = tNewSubscribeObj(); @@ -549,10 +606,16 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } char *key = mndMakeSubscribeKey(consumerGroup, newTopicName); + if (key == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } strcpy(pSub->key, key); + free(key); // set unassigned vg mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); // TODO: disable alter + create = true; } taosArrayPush(pSub->availConsumer, &consumerId); @@ -575,6 +638,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SSdbRaw *pRaw = mndSubActionEncode(pSub); sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); + if (!create) mndReleaseSubscribe(pMnode, pSub); #if 0 SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); if (pGroup == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3195691a13..1ecd76e1b5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -670,15 +670,13 @@ int tqItemSSize() { int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; - SRpcMsg rpcMsg; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; - int64_t reqOffset = pReq->offset; - int64_t fetchOffset = reqOffset; + int64_t fetchOffset = pReq->offset; int64_t blockingTime = pReq->blockingTime; int rspLen = 0; - SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 1, .pBlockData = NULL}; + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); ASSERT(pConsumer); @@ -690,6 +688,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (strcmp(pTopic->topicName, pReq->topic) != 0) { continue; } + rsp.committedOffset = pTopic->committedOffset; + rsp.reqOffset = pReq->offset; + rsp.skipLogNum = 0; if (fetchOffset == -1) { fetchOffset = pTopic->committedOffset + 1; @@ -715,6 +716,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (pHead->head.msgType == TDMT_VND_SUBMIT) { break; } + rsp.skipLogNum++; if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { atomic_store_8(&pTopic->buffer.output[pos].status, 0); skip = 1; @@ -745,6 +747,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } //TODO copy rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; + rsp.rspOffset = fetchOffset; atomic_store_8(&pTopic->buffer.output[pos].status, 0); @@ -752,6 +755,8 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { taosArrayDestroy(pRes); fetchOffset++; continue; + } else { + rsp.numOfTopics++; } rsp.pBlockData = pRes; @@ -931,6 +936,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { SMemRow row; int32_t kvIdx = 0; + int32_t curRow = 0; tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { // get all wanted col of that block @@ -940,8 +946,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { // TODO ASSERT(pCol->colId == pColData->info.colId); void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - memcpy(pColData->pData, val, pCol->bytes); + memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), val, pCol->bytes); } + curRow++; } return pArray; } From 8a180a33b2f063492bcea85510ed10989de5dc2c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 27 Jan 2022 16:53:40 +0800 Subject: [PATCH 06/42] fix query crash --- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2c445a59e3..80ff6fd8bf 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -233,7 +233,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { STransAction action = {0}; action.epSet = pCEp->epSet; action.pCont = buf; - action.contLen = tlen; + action.contLen = sizeof(SMsgHead) + tlen; action.msgType = TDMT_VND_MQ_SET_CONN; mndTransAppendRedoAction(pTrans, &action); From dfd077a4c9ef485f516a459e75a5680190ad16df Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 27 Jan 2022 17:32:45 +0800 Subject: [PATCH 07/42] fix hb issue --- source/client/inc/clientInt.h | 2 +- source/client/src/clientHb.c | 20 ++++++-------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 77249e3f0b..04e4978d4a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -227,7 +227,7 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f6c473333c..73ba59f772 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -430,22 +430,16 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { return pAppHbMgr; } -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { - if (NULL == pAppHbMgr) { - return; - } - +void appHbMgrCleanup(void) { pthread_mutex_lock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); - if (pAppHbMgr == pTarget) { - taosHashCleanup(pTarget->activeInfo); - pTarget->activeInfo = NULL; - taosHashCleanup(pTarget->connInfo); - pTarget->connInfo = NULL; - } + taosHashCleanup(pTarget->activeInfo); + pTarget->activeInfo = NULL; + taosHashCleanup(pTarget->connInfo); + pTarget->connInfo = NULL; } pthread_mutex_unlock(&clientHbMgr.lock); @@ -476,6 +470,7 @@ void hbMgrCleanUp() { if (old == 0) return; pthread_mutex_lock(&clientHbMgr.lock); + appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); pthread_mutex_unlock(&clientHbMgr.lock); @@ -535,9 +530,6 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { return; } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); - if (atomic_load_32(&pAppHbMgr->connKeyCnt) <= 0) { - appHbMgrCleanup(pAppHbMgr); - } } int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { From 963fb9b9f61040d3e8ac64b3070fdc4475e9d428 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 27 Jan 2022 18:05:23 +0800 Subject: [PATCH 08/42] fix mq rebalance --- source/client/src/clientImpl.c | 10 ++++++++-- source/dnode/mnode/impl/src/mndSubscribe.c | 21 ++++++++++++++++----- source/dnode/vnode/inc/vnode.h | 14 ++++++++++++++ source/dnode/vnode/src/tq/tq.c | 8 +++++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 56f89e30c4..227377c4a4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -628,6 +628,10 @@ struct tmq_message_t { }; int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { + if (code == -1) { + printf("discard\n"); + return 0; + } SMqClientVg* pVg = (SMqClientVg*)param; SMqConsumeRsp rsp; tDecodeSMqConsumeRsp(pMsg->pData, &rsp); @@ -680,6 +684,8 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; if (code != 0) { + + printf("exit wait %d\n", pParam->wait); if (pParam->wait) { tsem_post(&tmq->rspSem); } @@ -691,9 +697,9 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); int32_t sz = taosArrayGetSize(rsp.topics); // TODO: lock + printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size); + printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size); if (rsp.epoch != tmq->epoch) { - /*printf("rsp epoch %ld", rsp.epoch);*/ - /*printf("tmq epoch %ld", tmq->epoch);*/ //TODO if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 80ff6fd8bf..82173833ab 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -104,9 +104,6 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { if (found == 0) { taosArrayPush(pSub->availConsumer, &consumerId); } - SSdbRaw* pRaw = mndSubActionEncode(pSub); - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - sdbWriteNotFree(pMnode->pSdb, pRaw); int32_t assignedSz = taosArrayGetSize(pSub->assigned); topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); @@ -126,7 +123,9 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { taosArrayPush(rsp.topics, &topicEp); } if (changed || found) { - + SSdbRaw* pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWriteNotFree(pMnode->pSdb, pRaw); } mndReleaseSubscribe(pMnode, pSub); } @@ -183,12 +182,15 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } // TODO: acquire consumer, set status to unavail } +#if 0 SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId); pConsumer->epoch++; + printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size); SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer); sdbSetRawStatus(pRaw, SDB_STATUS_READY); sdbWriteNotFree(pMnode->pSdb, pRaw); mndReleaseConsumer(pMnode, pConsumer); +#endif } } if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0 && taosArrayGetSize(pSub->availConsumer) > 0) { @@ -207,6 +209,13 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { taosArrayPush(pSub->assigned, pCEp); pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); + SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId); + pConsumer->epoch++; + /*SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ + /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ + /*sdbWriteNotFree(pMnode->pSdb, pConsumerRaw);*/ + mndReleaseConsumer(pMnode, pConsumer); + // build msg SMqSetCVgReq req = {0}; @@ -216,8 +225,9 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { req.logicalPlan = pTopic->logicalPlan; req.physicalPlan = pTopic->physicalPlan; req.qmsg = pCEp->qmsg; + req.newConsumerId = consumerId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(tlen); + void *buf = malloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -262,6 +272,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas SArray *pArray; SArray *inner = taosArrayGet(pDag->pSubplans, 0); SSubplan *plan = taosArrayGetP(inner, 0); + plan->execNode.inUse = 0; strcpy(plan->execNode.epAddr[0].fqdn, "localhost"); plan->execNode.epAddr[0].port = 6030; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c8b47bf4a6..6fd7a99f54 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -71,6 +71,7 @@ typedef struct { typedef struct STqReadHandle { int64_t ver; uint64_t tbUid; + SHashObj* tbIdHash; SSubmitMsg* pMsg; SSubmitBlk* pBlock; SSubmitMsgIter msgIter; @@ -211,6 +212,19 @@ static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t t pHandle->tbUid = tbUid; } +static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle* pHandle, SArray* tbUidList) { + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + if (pHandle->tbIdHash == NULL) { + return -1; + } + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); + taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + //pHandle->tbUid = tbUid; + } + return 0; +} + void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle* pHandle); int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1ecd76e1b5..aaa04d6259 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -679,7 +679,13 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = -1; + rpcSendResponse(pMsg); + return 0; + } int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0; i < sz; i++) { From b0934caae1c1971a030a873c602a8a5d85c880ab Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 27 Jan 2022 18:08:34 +0800 Subject: [PATCH 09/42] add hash support for stable filter --- source/dnode/vnode/src/tq/tq.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aaa04d6259..d7fd02422b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -873,6 +873,11 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); return true; + } else if (pHandle->tbIdHash != NULL) { + void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); + if (ret != NULL) { + return true; + } } } return false; From 02a50b50e0bbf2169c57157d9c8e21d8437a9422 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Jan 2022 18:27:26 +0800 Subject: [PATCH 10/42] [td-11818] Update the API in topic-related function. --- source/dnode/vnode/inc/tsdb.h | 2 +- source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 24 ++++------- source/libs/executor/src/executorimpl.c | 57 +++++++++++++------------ 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 5f5acc1b05..af81cd2046 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -220,7 +220,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGro * clean up the query handle * @param queryHandle */ -void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); +void tsdbCleanupReadHandle(tsdbReadHandleT queryHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b56c5b30fa..ac5bde7af2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -204,8 +204,8 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SA pReadHandle->pColIdList = pColIdList; } -static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t tbUid) { - pHandle->tbUid = tbUid; +static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { +// pHandle->tbUid = pTableIdList; } void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1bca7aea10..964848f018 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -427,7 +427,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, return (tsdbReadHandleT)pReadHandle; _end: - tsdbCleanupQueryHandle(pReadHandle); + tsdbCleanupReadHandle(pReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } @@ -445,7 +445,7 @@ tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroup // todo apply the lastkey of table check to avoid to load header file pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList); if (pTsdbReadHandle->pTableCheckInfo == NULL) { -// tsdbCleanupQueryHandle(pTsdbReadHandle); +// tsdbCleanupReadHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } @@ -525,7 +525,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); if (pTsdbReadHandle->pTableCheckInfo == NULL) { -// tsdbCleanupQueryHandle(pTsdbReadHandle); +// tsdbCleanupReadHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -3051,7 +3051,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { // } // //out_of_memory: -// tsdbCleanupQueryHandle(pSecQueryHandle); +// tsdbCleanupReadHandle(pSecQueryHandle); // return terrno; //} @@ -3722,26 +3722,19 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch return terrno; } -#if 0 int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { - if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; - - STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - if (pTable == NULL) { + STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); + if (pTbCfg == NULL) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - tsdbUnlockRepoMeta(tsdb); goto _error; } - assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; - pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = pTable, .lastKey = startKey}; + STableKeyInfo info = {.lastKey = startKey, .uid = uid}; taosArrayPush(group, &info); taosArrayPush(pGroupInfo->pGroupList, &group); @@ -3751,6 +3744,7 @@ int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGr return terrno; } +#if 0 int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; @@ -3826,7 +3820,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { } -void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { +void tsdbCleanupReadHandle(tsdbReadHandleT queryHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; if (pTsdbReadHandle == NULL) { return; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 836b17a868..572da6a32c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2371,7 +2371,7 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); +// tsdbCleanupReadHandle(pRuntimeEnv->pTsdbReadHandle); pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; @@ -5453,7 +5453,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } -SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, uint64_t uid, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5476,7 +5476,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); - tqReadHandleSetTbUid(streamReadHandle, uid); + tqReadHandleSetTbUid(streamReadHandle, pTableIdList); pInfo->readerHandle = streamReadHandle; @@ -7749,20 +7749,21 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { } static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId); +static int32_t doCreateTableGroup(void* readerHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); @@ -7771,8 +7772,13 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); } else if (pPhyNode->info.type == OP_StreamScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. - return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo); + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. + STableGroupInfo groupInfo = {0}; + int32_t code = doCreateTableGroup(readerHandle, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + + SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); + ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); + return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pa, pTaskInfo); } } @@ -7810,29 +7816,24 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId); } +static int32_t doCreateTableGroup(void* readerHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { + int32_t code = 0; + if (tableType == TSDB_SUPER_TABLE) { + code = tsdbQuerySTableByTagCond(readerHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); + } else { // Create one table group. + code = tsdbGetOneTableGroup(readerHandle, tableUid, 0, pGroupInfo); + } + + return code; +} + static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId) { - int32_t code = 0; STableGroupInfo groupInfo = {0}; uint64_t uid = pTableScanNode->scan.uid; - STimeWindow window = pTableScanNode->window; - int32_t tableType = pTableScanNode->scan.tableType; - - if (tableType == TSDB_SUPER_TABLE) { - code = - tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId, taskId); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } else { // Create one table group. - groupInfo.numOfTables = 1; - groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES); - - SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - - STableKeyInfo info = {.lastKey = 0, .uid = uid}; - taosArrayPush(pa, &info); - taosArrayPush(groupInfo.pGroupList, &pa); + int32_t code = doCreateTableGroup(readerHandle, pTableScanNode->scan.tableType, uid, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + goto _error; } if (groupInfo.numOfTables == 0) { From cfc83751bb76306086dfad8c89eec28ff78e04a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Jan 2022 18:38:56 +0800 Subject: [PATCH 11/42] [td-11818] fix compiler error. --- source/client/src/clientImpl.c | 1 - source/libs/executor/src/executorimpl.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 63da5862d3..609ffb57ae 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,5 +1,4 @@ -#include "../../libs/scheduler/inc/schedulerInt.h" #include "clientInt.h" #include "clientLog.h" #include "parser.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 572da6a32c..5797837772 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7831,7 +7831,7 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo groupInfo = {0}; uint64_t uid = pTableScanNode->scan.uid; - int32_t code = doCreateTableGroup(readerHandle, pTableScanNode->scan.tableType, uid, queryId, taskId); + int32_t code = doCreateTableGroup(readerHandle, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); if (code != TSDB_CODE_SUCCESS) { goto _error; } From 8d71080146a8dee6184d782474d7b475ad75e7c0 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 27 Jan 2022 18:40:36 +0800 Subject: [PATCH 12/42] disable hb --- source/client/src/clientHb.c | 4 ++++ source/client/src/clientImpl.c | 6 +++--- source/client/src/clientMain.c | 4 ++-- source/client/src/clientMsgHandler.c | 2 +- source/client/test/clientTests.cpp | 10 ++++++---- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f6c473333c..6c8878921b 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -389,6 +389,7 @@ static void hbStopThread() { } SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { + return NULL; hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -452,6 +453,7 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { } int hbMgrInit() { + return 0; // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; @@ -469,6 +471,7 @@ int hbMgrInit() { } void hbMgrCleanUp() { + return; hbStopThread(); // destroy all appHbMgr @@ -506,6 +509,7 @@ int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { + return 0; SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; SHbConnInfo info = {0}; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 1378533949..07fbf34793 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -119,7 +119,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - p->pAppHbMgr = appHbMgrInit(p, key); + /*p->pAppHbMgr = appHbMgrInit(p, key);*/ taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -691,8 +691,8 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); int32_t sz = taosArrayGetSize(rsp.topics); // TODO: lock - printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size); - printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size); + /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ if (rsp.epoch != tmq->epoch) { //TODO if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index bdf3115981..b29813be03 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -78,7 +78,7 @@ void taos_close(TAOS* taos) { STscObj *pTscObj = (STscObj *)taos; tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); - taosRemoveRef(clientConnRefPool, pTscObj->id); + /*taosRemoveRef(clientConnRefPool, pTscObj->id);*/ } int taos_errno(TAOS_RES *tres) { @@ -343,4 +343,4 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { // TODO -} \ No newline at end of file +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 24d1c0f03c..6f74c29650 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -76,7 +76,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->connType = HEARTBEAT_TYPE_QUERY; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); + /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);*/ // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 3d57a5de1d..eec6a00405 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,6 +53,7 @@ TEST(testCase, driverInit_Test) { // taos_init(); } +#if 0 TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -562,7 +563,7 @@ TEST(testCase, insert_test) { taos_close(pConn); } -#if 0 +#endif TEST(testCase, create_topic_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -607,12 +608,11 @@ TEST(testCase, tmq_subscribe_Test) { tmq_subscribe(tmq, topic_list); while (1) { - tmq_message_t* msg = tmq_consume_poll(tmq, 0); - printf("get msg\n"); + tmq_message_t* msg = tmq_consume_poll(tmq, 1000); + //printf("get msg\n"); //if (msg == NULL) break; } } -#endif TEST(testCase, tmq_consume_Test) { } @@ -620,6 +620,7 @@ TEST(testCase, tmq_consume_Test) { TEST(testCase, tmq_commit_TEST) { } +#if 0 TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -732,5 +733,6 @@ TEST(testCase, agg_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif #pragma GCC diagnostic pop From d1dce49f14d84e0ab2fd9aadd49dc31d5b1697bb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Jan 2022 18:48:12 +0800 Subject: [PATCH 13/42] [td-11818] fix bug in topic . --- source/libs/executor/src/executorimpl.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5797837772..a0b16f4f20 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7778,7 +7778,19 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); - return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pa, pTaskInfo); + // Transfer the Array of STableKeyInfo into uid list. + size_t numOfTables = taosArrayGetSize(pa); + SArray* idList = taosArrayInit(numOfTables, sizeof(uint64_t)); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); + taosArrayPush(idList, &pkeyInfo->uid); + } + + SOperatorInfo* pOperator = createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, idList, pTaskInfo); + taosArrayDestroy(idList); + + //TODO destroy groupInfo + return pOperator; } } From 699b37cc18da3a25ef83f3c1220cbe56a9265626 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Jan 2022 19:03:58 +0800 Subject: [PATCH 14/42] [td-11818] fix bug in topic . --- source/dnode/vnode/inc/tsdb.h | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 20 ++++++++++---------- source/libs/executor/src/executorimpl.c | 9 +++++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index af81cd2046..e43e843218 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -138,7 +138,7 @@ bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); * @param reqId * @return */ -int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId); /** @@ -205,7 +205,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 964848f018..8965e16e40 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3635,29 +3635,29 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd // return TSDB_CODE_SUCCESS; //} -int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { - STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); + STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); if (pTbCfg == NULL) { - tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); +// tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; } if (pTbCfg->type != META_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); +// tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client goto _error; } //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - SSchemaWrapper* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - int32_t ret = getAllTableList(tsdb->pMeta, uid, res); + int32_t ret = getAllTableList(pMeta, uid, res); if (ret != TSDB_CODE_SUCCESS) { goto _error; } @@ -3665,8 +3665,8 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, - pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); +// tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, +// pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); taosArrayDestroy(res); return ret; @@ -3722,8 +3722,8 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch return terrno; } -int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { - STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); +int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { + STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); if (pTbCfg == NULL) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a0b16f4f20..665f4bc29c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7774,7 +7774,8 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask } else if (pPhyNode->info.type == OP_StreamScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. STableGroupInfo groupInfo = {0}; - int32_t code = doCreateTableGroup(readerHandle, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + + int32_t code = doCreateTableGroup(((STqReadHandle*)readerHandle)->pMeta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); @@ -7828,12 +7829,12 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId); } -static int32_t doCreateTableGroup(void* readerHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { +static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { - code = tsdbQuerySTableByTagCond(readerHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); + code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); } else { // Create one table group. - code = tsdbGetOneTableGroup(readerHandle, tableUid, 0, pGroupInfo); + code = tsdbGetOneTableGroup(metaHandle, tableUid, 0, pGroupInfo); } return code; From 55bb9cce20d8fcc08fbd6642ebdc1eb9e1c9d972 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 27 Jan 2022 11:28:13 -0500 Subject: [PATCH 15/42] TD-13197 SELECT statement syntax definition --- include/common/ttokendef.h | 100 +- include/nodes/nodes.h | 37 +- source/libs/parser/inc/astCreateContext.h | 8 - source/libs/parser/inc/astCreateFuncs.h | 13 +- source/libs/parser/inc/new_sql.y | 249 +-- source/libs/parser/src/astCreateContext.c | 41 +- source/libs/parser/src/astCreateFuncs.c | 146 +- source/libs/parser/src/new_sql.c | 1757 ++++++++++++--------- source/libs/parser/src/parserImpl.c | 8 +- source/libs/parser/test/newParserTest.cpp | 28 +- source/nodes/src/nodesEqualFuncs.c | 2 +- source/nodes/src/nodesUtilFuncs.c | 7 +- 12 files changed, 1362 insertions(+), 1034 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index f8aa7df02a..b2a64bfac4 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -209,48 +209,76 @@ #define NEW_TK_OR 1 #define NEW_TK_AND 2 -#define NEW_TK_NOT 3 -#define NEW_TK_UNION 4 -#define NEW_TK_ALL 5 -#define NEW_TK_MINUS 6 -#define NEW_TK_EXCEPT 7 -#define NEW_TK_INTERSECT 8 -#define NEW_TK_NK_PLUS 9 -#define NEW_TK_NK_MINUS 10 -#define NEW_TK_NK_STAR 11 -#define NEW_TK_NK_SLASH 12 +#define NEW_TK_UNION 3 +#define NEW_TK_ALL 4 +#define NEW_TK_MINUS 5 +#define NEW_TK_EXCEPT 6 +#define NEW_TK_INTERSECT 7 +#define NEW_TK_NK_PLUS 8 +#define NEW_TK_NK_MINUS 9 +#define NEW_TK_NK_STAR 10 +#define NEW_TK_NK_SLASH 11 +#define NEW_TK_NK_REM 12 #define NEW_TK_SHOW 13 #define NEW_TK_DATABASES 14 #define NEW_TK_NK_INTEGER 15 #define NEW_TK_NK_FLOAT 16 #define NEW_TK_NK_STRING 17 #define NEW_TK_NK_BOOL 18 -#define NEW_TK_NK_NOW 19 -#define NEW_TK_NK_ID 20 -#define NEW_TK_NK_QUESTION 21 -#define NEW_TK_NK_LP 22 -#define NEW_TK_NK_RP 23 -#define NEW_TK_NK_DOT 24 -#define NEW_TK_FROM 25 -#define NEW_TK_NK_COMMA 26 -#define NEW_TK_AS 27 -#define NEW_TK_JOIN 28 -#define NEW_TK_ON 29 -#define NEW_TK_INNER 30 -#define NEW_TK_SELECT 31 -#define NEW_TK_DISTINCT 32 -#define NEW_TK_ORDER 33 -#define NEW_TK_BY 34 -#define NEW_TK_SLIMIT 35 -#define NEW_TK_SOFFSET 36 -#define NEW_TK_LIMIT 37 -#define NEW_TK_OFFSET 38 -#define NEW_TK_NK_LR 39 -#define NEW_TK_ASC 40 -#define NEW_TK_DESC 41 -#define NEW_TK_NULLS 42 -#define NEW_TK_FIRST 43 -#define NEW_TK_LAST 44 +#define NEW_TK_TIMESTAMP 19 +#define NEW_TK_NK_VARIABLE 20 +#define NEW_TK_NK_COMMA 21 +#define NEW_TK_NK_ID 22 +#define NEW_TK_NK_LP 23 +#define NEW_TK_NK_RP 24 +#define NEW_TK_NK_DOT 25 +#define NEW_TK_BETWEEN 26 +#define NEW_TK_NOT 27 +#define NEW_TK_IS 28 +#define NEW_TK_NULL 29 +#define NEW_TK_NK_LT 30 +#define NEW_TK_NK_GT 31 +#define NEW_TK_NK_LE 32 +#define NEW_TK_NK_GE 33 +#define NEW_TK_NK_NE 34 +#define NEW_TK_NK_EQ 35 +#define NEW_TK_LIKE 36 +#define NEW_TK_MATCH 37 +#define NEW_TK_NMATCH 38 +#define NEW_TK_IN 39 +#define NEW_TK_FROM 40 +#define NEW_TK_AS 41 +#define NEW_TK_JOIN 42 +#define NEW_TK_ON 43 +#define NEW_TK_INNER 44 +#define NEW_TK_SELECT 45 +#define NEW_TK_DISTINCT 46 +#define NEW_TK_WHERE 47 +#define NEW_TK_PARTITION 48 +#define NEW_TK_BY 49 +#define NEW_TK_SESSION 50 +#define NEW_TK_STATE_WINDOW 51 +#define NEW_TK_INTERVAL 52 +#define NEW_TK_SLIDING 53 +#define NEW_TK_FILL 54 +#define NEW_TK_VALUE 55 +#define NEW_TK_NONE 56 +#define NEW_TK_PREV 57 +#define NEW_TK_LINEAR 58 +#define NEW_TK_NEXT 59 +#define NEW_TK_GROUP 60 +#define NEW_TK_HAVING 61 +#define NEW_TK_ORDER 62 +#define NEW_TK_SLIMIT 63 +#define NEW_TK_SOFFSET 64 +#define NEW_TK_LIMIT 65 +#define NEW_TK_OFFSET 66 +#define NEW_TK_NK_LR 67 +#define NEW_TK_ASC 68 +#define NEW_TK_DESC 69 +#define NEW_TK_NULLS 70 +#define NEW_TK_FIRST 71 +#define NEW_TK_LAST 72 #define TK_SPACE 300 #define TK_COMMENT 301 diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index 5fb4c3476d..39638b3757 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -51,6 +51,8 @@ typedef enum ENodeType { QUERY_NODE_STATE_WINDOW, QUERY_NODE_SESSION_WINDOW, QUERY_NODE_INTERVAL_WINDOW, + QUERY_NODE_NODE_LIST, + QUERY_NODE_FILL, QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, @@ -157,14 +159,19 @@ typedef struct SLogicConditionNode { typedef struct SIsNullCondNode { ENodeType type; // QUERY_NODE_IS_NULL_CONDITION SNode* pExpr; - bool isNot; + bool isNull; } SIsNullCondNode; +typedef struct SNodeListNode { + ENodeType type; // QUERY_NODE_NODE_LIST + SNodeList* pNodeList; +} SNodeListNode; + typedef struct SFunctionNode { SExprNode type; // QUERY_NODE_FUNCTION char functionName[TSDB_FUNC_NAME_LEN]; int32_t funcId; - SNodeList* pParameterList; // SNode + SNodeList* pParameterList; } SFunctionNode; typedef struct STableNode { @@ -242,11 +249,27 @@ typedef struct SSessionWindowNode { typedef struct SIntervalWindowNode { ENodeType type; // QUERY_NODE_INTERVAL_WINDOW - int64_t interval; - int64_t sliding; - int64_t offset; + SNode* pInterval; // SValueNode + SNode* pOffset; // SValueNode + SNode* pSliding; // SValueNode + SNode* pFill; } SIntervalWindowNode; +typedef enum EFillMode { + FILL_MODE_NONE = 1, + FILL_MODE_VALUE, + FILL_MODE_PREV, + FILL_MODE_NULL, + FILL_MODE_LINEAR, + FILL_MODE_NEXT +} EFillMode; + +typedef struct SFillNode { + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode +} SFillNode; + typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; @@ -259,8 +282,8 @@ typedef struct SSelectStmt { SNodeList* pGroupByList; // SGroupingSetNode SNode* pHaving; SNodeList* pOrderByList; // SOrderByExprNode - SLimitNode* pLimit; - SLimitNode* pSlimit; + SNode* pLimit; + SNode* pSlimit; } SSelectStmt; typedef enum ESetOperatorType { diff --git a/source/libs/parser/inc/astCreateContext.h b/source/libs/parser/inc/astCreateContext.h index 107802bbc4..5458500a82 100644 --- a/source/libs/parser/inc/astCreateContext.h +++ b/source/libs/parser/inc/astCreateContext.h @@ -23,11 +23,6 @@ extern "C" { #include "nodes.h" #include "parser.h" -typedef enum EResourceType { - AST_CXT_RESOURCE_NODE = 1, - AST_CXT_RESOURCE_NODE_LIST -} EResourceType; - typedef struct SAstCreateContext { SParseContext* pQueryCxt; bool notSupport; @@ -39,9 +34,6 @@ typedef struct SAstCreateContext { int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt); int32_t destroyAstCreateContext(SAstCreateContext* pCxt); -void* acquireRaii(SAstCreateContext* pCxt, EResourceType type, void* p); -void* releaseRaii(SAstCreateContext* pCxt, void* p); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index 515df7f7fe..c31160a27e 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -32,14 +32,25 @@ SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode); SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull); +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); -SNode* createLimitNode(SAstCreateContext* pCxt, SNode* pLimit, SNode* pOffset); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index 0fc316dd29..451aaddc9e 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -22,8 +22,15 @@ #include "ttokendef.h" #include "astCreateFuncs.h" +#if 0 #define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) #define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif } %syntax_error { @@ -44,43 +51,37 @@ pCxt->valid = false; } -%parse_accept { printf("parsing complete!\n" );} +%parse_accept { PARSER_COMPLETE; } %left OR. %left AND. -%right NOT. %left UNION ALL MINUS EXCEPT INTERSECT. -//%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -//%left GT GE LT LE. //%left BITAND BITOR LSHIFT RSHIFT. %left NK_PLUS NK_MINUS. //%left DIVIDE TIMES. -%left NK_STAR NK_SLASH. //REM. +%left NK_STAR NK_SLASH NK_REM. //%left CONCAT. //%right UMINUS UPLUS BITNOT. cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } -/*5.4*********************************************** literal *********************************************************/ -unsigned_integer(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } -unsigned_approximate_numeric(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +/************************************************ literal *************************************************************/ +literal(A) ::= NK_INTEGER(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +literal(A) ::= NK_FLOAT(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +literal(A) ::= NK_STRING(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } +literal(A) ::= NK_BOOL(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } +literal(A) ::= TIMESTAMP NK_STRING(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } +literal(A) ::= duration_literal(B). { PARSER_TRACE; A = B; } -signed_integer(A) ::= unsigned_integer(B). { A = B; } -signed_integer(A) ::= NK_PLUS unsigned_integer(B). { A = B; } -signed_integer(A) ::= NK_MINUS unsigned_integer(B). { A = addMinusSign(pCxt, B);} +duration_literal(A) ::= NK_VARIABLE(B). { PARSER_TRACE; A = createDurationValueNode(pCxt, &B); } -unsigned_literal ::= unsigned_numeric_literal. -unsigned_literal ::= general_literal. +%type literal_list { SNodeList* } +%destructor literal_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +literal_list(A) ::= literal(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -unsigned_numeric_literal ::= unsigned_integer. -unsigned_numeric_literal ::= unsigned_approximate_numeric. - -general_literal ::= NK_STRING. -general_literal ::= NK_BOOL. -general_literal ::= NK_NOW. - -/*5.4*********************************************** names and identifiers *********************************************************/ +/************************************************ names and identifiers ***********************************************/ %type db_name { SToken } %destructor db_name { PARSER_DESTRUCTOR_TRACE; } db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } @@ -93,29 +94,76 @@ table_name(A) ::= NK_ID(B). %destructor column_name { PARSER_DESTRUCTOR_TRACE; } column_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } +%type function_name { SToken } +%destructor function_name { PARSER_DESTRUCTOR_TRACE; } +function_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + %type table_alias { SToken } %destructor table_alias { PARSER_DESTRUCTOR_TRACE; } table_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -/*6.4*********************************************** value_specification *********************************************************/ -unsigned_value_specification ::= unsigned_literal. -unsigned_value_specification ::= NK_QUESTION. +%type column_alias { SToken } +%destructor column_alias { PARSER_DESTRUCTOR_TRACE; } +column_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -/*6.35 todo *********************************************** value_expression_primary *********************************************************/ -value_expression_primary ::= NK_LP value_expression NK_RP. -value_expression_primary ::= nonparenthesized_value_expression_primary. +/************************************************ expression **********************************************************/ +expression(A) ::= literal(B). { PARSER_TRACE; A = B; } +//expression(A) ::= NK_QUESTION(B). { PARSER_TRACE; A = B; } +//expression(A) ::= pseudo_column(B). { PARSER_TRACE; A = B; } +expression(A) ::= column_reference(B). { PARSER_TRACE; A = B; } +expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { PARSER_TRACE; A = createFunctionNode(pCxt, &B, C); } +//expression(A) ::= cast_expression(B). { PARSER_TRACE; A = B; } +//expression(A) ::= case_expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= subquery(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_LP expression(B) NK_RP. { PARSER_TRACE; A = B; } +expression(A) ::= NK_PLUS expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_MINUS expression(B). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, NULL); } +expression(A) ::= expression(B) NK_PLUS expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_ADD, B, C); } +expression(A) ::= expression(B) NK_MINUS expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, C); } +expression(A) ::= expression(B) NK_STAR expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MULTI, B, C); } +expression(A) ::= expression(B) NK_SLASH expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_DIV, B, C); } +expression(A) ::= expression(B) NK_REM expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MOD, B, C); } -nonparenthesized_value_expression_primary ::= unsigned_value_specification. -nonparenthesized_value_expression_primary ::= column_reference. -//nonparenthesized_value_expression_primary ::= agg_function. -nonparenthesized_value_expression_primary ::= subquery. -//nonparenthesized_value_expression_primary ::= case_expression. // todo -//nonparenthesized_value_expression_primary ::= cast_specification. // todo +%type expression_list { SNodeList* } +%destructor expression_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +expression_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } column_reference(A) ::= column_name(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } column_reference(A) ::= table_name(B) NK_DOT column_name(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } -/*6.35*********************************************** boolean_value_expression *********************************************************/ +//pseudo_column(A) ::= NK_NOW. { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); } + +/************************************************ predicate ***********************************************************/ +predicate(A) ::= expression(B) compare_op(C) expression(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); } +//predicate(A) ::= expression(B) compare_op sub_type expression(B). +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createBetweenAnd(pCxt, B, C, D); } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createNotBetweenAnd(pCxt, C, B, D); } +predicate(A) ::= expression(B) IS NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, true); } +predicate(A) ::= expression(B) IS NOT NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, false); } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); } + +%type compare_op { EOperatorType } +%destructor compare_op { PARSER_DESTRUCTOR_TRACE; } +compare_op(A) ::= NK_LT. { PARSER_TRACE; A = OP_TYPE_LOWER_THAN; } +compare_op(A) ::= NK_GT. { PARSER_TRACE; A = OP_TYPE_GREATER_THAN; } +compare_op(A) ::= NK_LE. { PARSER_TRACE; A = OP_TYPE_LOWER_EQUAL; } +compare_op(A) ::= NK_GE. { PARSER_TRACE; A = OP_TYPE_GREATER_EQUAL; } +compare_op(A) ::= NK_NE. { PARSER_TRACE; A = OP_TYPE_NOT_EQUAL; } +compare_op(A) ::= NK_EQ. { PARSER_TRACE; A = OP_TYPE_EQUAL; } +compare_op(A) ::= LIKE. { PARSER_TRACE; A = OP_TYPE_LIKE; } +compare_op(A) ::= NOT LIKE. { PARSER_TRACE; A = OP_TYPE_NOT_LIKE; } +compare_op(A) ::= MATCH. { PARSER_TRACE; A = OP_TYPE_MATCH; } +compare_op(A) ::= NMATCH. { PARSER_TRACE; A = OP_TYPE_NMATCH; } + +%type in_op { EOperatorType } +%destructor in_op { PARSER_DESTRUCTOR_TRACE; } +in_op(A) ::= IN. { PARSER_TRACE; A = OP_TYPE_IN; } +in_op(A) ::= NOT IN. { PARSER_TRACE; A = OP_TYPE_NOT_IN; } + +in_predicate_value(A) ::= NK_LP expression_list(B) NK_RP. { PARSER_TRACE; A = createNodeListNode(pCxt, B); } + +/************************************************ boolean_value_expression ********************************************/ boolean_value_expression(A) ::= boolean_primary(B). { PARSER_TRACE; A = B; } boolean_value_expression(A) ::= NOT boolean_primary(B). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); } boolean_value_expression(A) ::= @@ -126,32 +174,31 @@ boolean_value_expression(A) ::= boolean_primary(A) ::= predicate(B). { PARSER_TRACE; A = B; } boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP. { PARSER_TRACE; A = B; } -/*7.5*********************************************** from_clause *********************************************************/ +/************************************************ from_clause *********************************************************/ from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } -/*7.6*********************************************** table_reference *****************************************************/ +/************************************************ table_reference *****************************************************/ table_reference(A) ::= table_primary(B). { PARSER_TRACE; A = B; } table_reference(A) ::= joined_table(B). { PARSER_TRACE; A = B; } -table_primary(A) ::= table_name(B) correlation_or_recognition_opt(C). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); } -table_primary(A) ::= - db_name(B) NK_DOT table_name(C) correlation_or_recognition_opt(D). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); } -table_primary(A) ::= subquery(B) correlation_or_recognition_opt(C). { PARSER_TRACE; A = createTempTableNode(pCxt, B, &C); } +table_primary(A) ::= table_name(B) alias_opt(C). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) alias_opt(C). { PARSER_TRACE; A = createTempTableNode(pCxt, B, &C); } table_primary ::= parenthesized_joined_table. -%type correlation_or_recognition_opt { SToken } -%destructor correlation_or_recognition_opt { PARSER_DESTRUCTOR_TRACE; } -correlation_or_recognition_opt(A) ::= . { PARSER_TRACE; A = nil_token; } -correlation_or_recognition_opt(A) ::= table_alias(B). { PARSER_TRACE; A = B; } -correlation_or_recognition_opt(A) ::= AS table_alias(B). { PARSER_TRACE; A = B; } +%type alias_opt { SToken } +%destructor alias_opt { PARSER_DESTRUCTOR_TRACE; } +alias_opt(A) ::= . { PARSER_TRACE; A = nil_token; } +alias_opt(A) ::= table_alias(B). { PARSER_TRACE; A = B; } +alias_opt(A) ::= AS table_alias(B). { PARSER_TRACE; A = B; } parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { PARSER_TRACE; A = B; } parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { PARSER_TRACE; A = B; } -/*7.10*********************************************** joined_table ************************************************************/ +/************************************************ joined_table ********************************************************/ joined_table(A) ::= table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { PARSER_TRACE; A = createJoinTableNode(pCxt, C, B, D, E); } @@ -159,7 +206,7 @@ joined_table(A) ::= %destructor join_type { PARSER_DESTRUCTOR_TRACE; } join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } -/*7.15*********************************************** query_specification *************************************************/ +/************************************************ query_specification *************************************************/ query_specification(A) ::= SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) partition_by_clause_opt(F) twindow_clause_opt(G) @@ -189,12 +236,53 @@ select_list(A) ::= select_sublist(B). select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; } -select_item(A) ::= value_expression(B) NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } -select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= expression(B). { PARSER_TRACE; A = B; } +select_item(A) ::= expression(B) column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= expression(B) AS column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } -/*7.16*********************************************** query_expression ****************************************************/ +where_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +where_clause_opt(A) ::= WHERE search_condition(B). { PARSER_TRACE; A = B; } + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +partition_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { PARSER_TRACE; A = B; } + +twindow_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +twindow_clause_opt(A) ::= + SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { PARSER_TRACE; A = createSessionWindowNode(pCxt, B, &C); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { PARSER_TRACE; A = createStateWindowNode(pCxt, B); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, NULL, C, D); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + sliding_opt(D) fill_opt(E). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, C, D, E); } + +sliding_opt(A) ::= . { PARSER_TRACE; A = NULL; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { PARSER_TRACE; A = B; } + +fill_opt(A) ::= . { PARSER_TRACE; A = NULL; } +fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, B, NULL); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } + +%type fill_mode { EFillMode } +%destructor fill_mode { PARSER_DESTRUCTOR_TRACE; } +fill_mode(A) ::= NONE. { PARSER_TRACE; A = FILL_MODE_NONE; } +fill_mode(A) ::= PREV. { PARSER_TRACE; A = FILL_MODE_PREV; } +fill_mode(A) ::= NULL. { PARSER_TRACE; A = FILL_MODE_NULL; } +fill_mode(A) ::= LINEAR. { PARSER_TRACE; A = FILL_MODE_LINEAR; } +fill_mode(A) ::= NEXT. { PARSER_TRACE; A = FILL_MODE_NEXT; } + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; } + +having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; } + +/************************************************ query_expression ****************************************************/ query_expression(A) ::= query_expression_body(B) order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { @@ -219,25 +307,22 @@ order_by_clause_opt(A) ::= . order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } slimit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -slimit_clause_opt(A) ::= SLIMIT signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, 0); } -slimit_clause_opt(A) ::= SLIMIT signed_integer(B) SOFFSET signed_integer(C). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } -slimit_clause_opt(A) ::= SLIMIT signed_integer(C) NK_COMMA signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } limit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -limit_clause_opt(A) ::= LIMIT signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, 0); } -limit_clause_opt(A) ::= LIMIT signed_integer(B) OFFSET signed_integer(C). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } -limit_clause_opt(A) ::= LIMIT signed_integer(C) NK_COMMA signed_integer(B). { PARSER_TRACE; A = createLimitNode(pCxt, B, C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } -/*7.18*********************************************** subquery ************************************************************/ +/************************************************ subquery ************************************************************/ subquery(A) ::= NK_LR query_expression(B) NK_RP. { PARSER_TRACE; A = B; } -/*8.1*********************************************** predicate ************************************************************/ -predicate ::= . - -/*8.21 todo *********************************************** search_condition ************************************************************/ +/************************************************ search_condition ****************************************************/ search_condition(A) ::= boolean_value_expression(B). { PARSER_TRACE; A = B; } -/*10.10*********************************************** sort_specification_list ************************************************************/ +/************************************************ sort_specification_list *********************************************/ %type sort_specification_list { SNodeList* } %destructor sort_specification_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } @@ -245,7 +330,7 @@ sort_specification_list(A) ::= sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } sort_specification(A) ::= - value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } + expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } %type ordering_specification_opt EOrder %destructor ordering_specification_opt { PARSER_DESTRUCTOR_TRACE; } @@ -258,39 +343,3 @@ ordering_specification_opt(A) ::= DESC. null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } - -/************************************************ todo ************************************************************/ - -where_clause_opt ::= . - -%type partition_by_clause_opt { SNodeList* } -%destructor partition_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } -partition_by_clause_opt ::=. - -twindow_clause_opt ::= . - -%type group_by_clause_opt { SNodeList* } -%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } -group_by_clause_opt ::= . - -having_clause_opt ::= . - -//////////////////////// value_function ///////////////////////////////// -value_function ::= NK_ID NK_LP value_expression NK_RP. -value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP. - -//////////////////////// value_expression ///////////////////////////////// -value_expression ::= common_value_expression. - -common_value_expression ::= numeric_value_expression. - -numeric_value_expression ::= numeric_primary. -numeric_value_expression ::= NK_PLUS numeric_primary. -numeric_value_expression ::= NK_MINUS numeric_primary. -numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression. - -numeric_primary ::= value_expression_primary. -numeric_primary ::= value_function. \ No newline at end of file diff --git a/source/libs/parser/src/astCreateContext.c b/source/libs/parser/src/astCreateContext.c index 6e57683afd..a8c82780ca 100644 --- a/source/libs/parser/src/astCreateContext.c +++ b/source/libs/parser/src/astCreateContext.c @@ -16,53 +16,14 @@ #include "ttoken.h" #include "astCreateContext.h" -typedef struct SResourceEntry { - EResourceType type; - void* res; -} SResourceEntry; - int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt) { pCxt->pQueryCxt = pQueryCxt; pCxt->notSupport = false; pCxt->valid = true; pCxt->pRootNode = NULL; - pCxt->pResourceHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (NULL == pCxt->pResourceHash) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } return TSDB_CODE_SUCCESS; } int32_t destroyAstCreateContext(SAstCreateContext* pCxt) { - SResourceEntry* item = taosHashIterate(pCxt->pResourceHash, NULL); - while (item) { - switch (item->type) { - case AST_CXT_RESOURCE_NODE: - nodesDestroyNode(item->res); - break; - case AST_CXT_RESOURCE_NODE_LIST: - nodesDestroyList(item->res); - break; - default: - tfree(item->res); - } - item = taosHashIterate(pCxt->pResourceHash, item); - } -} - -void* acquireRaii(SAstCreateContext* pCxt, EResourceType type, void* p) { - if (NULL == p) { - return NULL; - } - SResourceEntry entry = { .type = type, .res = p }; - taosHashPut(pCxt->pResourceHash, &p, POINTER_BYTES, &entry, sizeof(SResourceEntry)); - return p; -} - -void* releaseRaii(SAstCreateContext* pCxt, void* p) { - if (NULL == p) { - return NULL; - } - taosHashRemove(pCxt->pResourceHash, &p, POINTER_BYTES); - return p; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index 8c67fc1690..f054649e1a 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -53,11 +53,11 @@ static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { SNodeList* list = nodesMakeList(); CHECK_OUT_OF_MEM(list); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE_LIST, nodesListAppend(list, releaseRaii(pCxt, pNode))); + return nodesListAppend(list, pNode); } SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { - return nodesListAppend(pList, releaseRaii(pCxt, pNode)); + return nodesListAppend(pList, pNode); } SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { @@ -70,14 +70,21 @@ SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const strncpy(col->tableName, pTableName->z, pTableName->n); } strncpy(col->colName, pColumnName->z, pColumnName->n); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, col); + return (SNode*)col; } SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) { SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); CHECK_OUT_OF_MEM(val); - // strncpy(col->colName, pColumnName->z, pColumnName->n); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, val); + // todo + return (SNode*)val; +} + +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + // todo + return (SNode*)val; } SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode) { @@ -89,9 +96,51 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ CHECK_OUT_OF_MEM(cond); cond->condType = type; cond->pParameterList = nodesMakeList(); - nodesListAppend(cond->pParameterList, releaseRaii(pCxt, pParam1)); - nodesListAppend(cond->pParameterList, releaseRaii(pCxt, pParam2)); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, cond); + nodesListAppend(cond->pParameterList, pParam1); + nodesListAppend(cond->pParameterList, pParam2); + return (SNode*)cond; +} + +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight) { + SOperatorNode* op = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + CHECK_OUT_OF_MEM(op); + op->opType = type; + op->pLeft = pLeft; + op->pRight = pRight; + return (SNode*)op; +} + +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, + createOperatorNode(pCxt, OP_TYPE_GREATER_EQUAL, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_LOWER_EQUAL, pExpr, pRight)); +} + +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, + createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pExpr, pRight)); +} + +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull) { + SIsNullCondNode* cond = (SIsNullCondNode*)nodesMakeNode(QUERY_NODE_IS_NULL_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->pExpr = pExpr; + cond->isNull = isNull; + return (SNode*)cond; +} + +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { + SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + CHECK_OUT_OF_MEM(func); + strncpy(func->functionName, pFuncName->z, pFuncName->n); + func->pParameterList = pParameterList; + return (SNode*)func; +} + +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) { + SNodeListNode* list = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + CHECK_OUT_OF_MEM(list); + list->pNodeList = pList; + return (SNode*)list; } SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias) { @@ -104,41 +153,74 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const strncpy(realTable->dbName, pDbName->z, pDbName->n); } strncpy(realTable->table.tableName, pTableName->z, pTableName->n); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, realTable); + return (SNode*)realTable; } SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias) { STempTableNode* tempTable = (STempTableNode*)nodesMakeNode(QUERY_NODE_TEMP_TABLE); CHECK_OUT_OF_MEM(tempTable); tempTable->pSubquery = pSubquery; - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, tempTable); + return (SNode*)tempTable; } SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond) { SJoinTableNode* joinTable = (SJoinTableNode*)nodesMakeNode(QUERY_NODE_JOIN_TABLE); CHECK_OUT_OF_MEM(joinTable); joinTable->joinType = type; - joinTable->pLeft = releaseRaii(pCxt, pLeft); - joinTable->pRight = releaseRaii(pCxt, pRight); + joinTable->pLeft = pLeft; + joinTable->pRight = pRight; joinTable->pOnCond = pJoinCond; - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, joinTable); + return (SNode*)joinTable; } -SNode* createLimitNode(SAstCreateContext* pCxt, SNode* pLimit, SNode* pOffset) { +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); CHECK_OUT_OF_MEM(limitNode); // limitNode->limit = limit; // limitNode->offset = offset; - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, limitNode); + return (SNode*)limitNode; } SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { SOrderByExprNode* orderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); CHECK_OUT_OF_MEM(orderByExpr); - orderByExpr->pExpr = releaseRaii(pCxt, pExpr); + orderByExpr->pExpr = pExpr; orderByExpr->order = order; orderByExpr->nullOrder = nullOrder; - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, orderByExpr); + return (SNode*)orderByExpr; +} + +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal) { + SSessionWindowNode* session = (SSessionWindowNode*)nodesMakeNode(QUERY_NODE_SESSION_WINDOW); + CHECK_OUT_OF_MEM(session); + session->pCol = pCol; + // session->gap = getInteger(pVal); + return (SNode*)session; +} + +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { + SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); + CHECK_OUT_OF_MEM(state); + state->pCol = pCol; + return (SNode*)state; +} + +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { + SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + CHECK_OUT_OF_MEM(interval); + interval->pInterval = pInterval; + interval->pOffset = pOffset; + interval->pSliding = pSliding; + interval->pFill = pFill; + return (SNode*)interval; +} + +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { + SFillNode* fill = (SFillNode*)nodesMakeNode(QUERY_NODE_FILL); + CHECK_OUT_OF_MEM(fill); + fill->mode = mode; + fill->pValues = pValues; + return (SNode*)fill; } SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { @@ -148,56 +230,56 @@ SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* p SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pWhere = releaseRaii(pCxt, pWhere); + ((SSelectStmt*)pStmt)->pWhere = pWhere; } return pStmt; } SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pPartitionByList = releaseRaii(pCxt, pPartitionByList); + ((SSelectStmt*)pStmt)->pPartitionByList = pPartitionByList; } return pStmt; } SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pWindow = releaseRaii(pCxt, pWindow); + ((SSelectStmt*)pStmt)->pWindow = pWindow; } return pStmt; } SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pGroupByList = releaseRaii(pCxt, pGroupByList); + ((SSelectStmt*)pStmt)->pGroupByList = pGroupByList; } return pStmt; } SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pHaving = releaseRaii(pCxt, pHaving); + ((SSelectStmt*)pStmt)->pHaving = pHaving; } return pStmt; } SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pOrderByList = releaseRaii(pCxt, pOrderByList); + ((SSelectStmt*)pStmt)->pOrderByList = pOrderByList; } return pStmt; } SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pSlimit = releaseRaii(pCxt, pSlimit); + ((SSelectStmt*)pStmt)->pSlimit = pSlimit; } return pStmt; } SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pLimit = releaseRaii(pCxt, pLimit); + ((SSelectStmt*)pStmt)->pLimit = pLimit; } return pStmt; } @@ -209,23 +291,23 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr if (NULL == pProjectionList) { select->isStar = true; } - select->pProjectionList = releaseRaii(pCxt, pProjectionList); - select->pFromTable = releaseRaii(pCxt, pTable); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, select); + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + return (SNode*)select; } SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) { SSetOperator* setOp = (SSetOperator*)nodesMakeNode(QUERY_NODE_SET_OPERATOR); CHECK_OUT_OF_MEM(setOp); setOp->opType = type; - setOp->pLeft = releaseRaii(pCxt, pLeft); - setOp->pRight = releaseRaii(pCxt, pRight); - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, setOp); + setOp->pLeft = pLeft; + setOp->pRight = pRight; + return (SNode*)setOp; } SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) { SShowStmt* show = (SShowStmt*)nodesMakeNode(QUERY_NODE_SHOW_STMT); CHECK_OUT_OF_MEM(show); show->showType = type; - return acquireRaii(pCxt, AST_CXT_RESOURCE_NODE, show); + return (SNode*)show; } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index f5eedc9557..20f82c150a 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -37,8 +37,15 @@ #include "ttokendef.h" #include "astCreateFuncs.h" +#if 0 #define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) #define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -102,19 +109,21 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 98 +#define YYNOCODE 125 #define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - SNodeList* yy40; - EOrder yy106; - EJoinType yy148; - SNode* yy168; - SToken yy169; - bool yy173; - ENullOrder yy193; + EOperatorType yy40; + EFillMode yy44; + SToken yy79; + ENullOrder yy107; + EJoinType yy162; + SNodeList* yy174; + EOrder yy188; + SNode* yy212; + bool yy237; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -129,17 +138,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 92 -#define YYNRULE 106 -#define YYNTOKEN 45 -#define YY_MAX_SHIFT 91 -#define YY_MIN_SHIFTREDUCE 169 -#define YY_MAX_SHIFTREDUCE 274 -#define YY_ERROR_ACTION 275 -#define YY_ACCEPT_ACTION 276 -#define YY_NO_ACTION 277 -#define YY_MIN_REDUCE 278 -#define YY_MAX_REDUCE 383 +#define YYNSTATE 149 +#define YYNRULE 135 +#define YYNTOKEN 73 +#define YY_MAX_SHIFT 148 +#define YY_MIN_SHIFTREDUCE 245 +#define YY_MAX_SHIFTREDUCE 379 +#define YY_ERROR_ACTION 380 +#define YY_ACCEPT_ACTION 381 +#define YY_NO_ACTION 382 +#define YY_MIN_REDUCE 383 +#define YY_MAX_REDUCE 517 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -206,142 +215,199 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (464) +#define YY_ACTTAB_COUNT (656) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 60, 60, 77, 60, 60, 60, 290, 80, 289, 47, - /* 10 */ 60, 60, 62, 60, 60, 60, 171, 172, 245, 246, - /* 20 */ 247, 70, 249, 6, 21, 20, 76, 33, 292, 295, - /* 30 */ 46, 11, 10, 9, 8, 31, 72, 318, 34, 22, - /* 40 */ 19, 42, 300, 75, 49, 74, 187, 60, 62, 60, - /* 50 */ 60, 60, 60, 179, 60, 60, 60, 22, 87, 289, - /* 60 */ 44, 60, 60, 37, 60, 60, 60, 18, 202, 63, - /* 70 */ 291, 295, 294, 291, 295, 73, 291, 295, 60, 60, - /* 80 */ 310, 60, 60, 60, 262, 80, 289, 5, 60, 60, - /* 90 */ 62, 60, 60, 60, 83, 341, 51, 50, 60, 37, - /* 100 */ 60, 60, 171, 79, 53, 15, 60, 60, 306, 60, - /* 110 */ 60, 60, 1, 87, 289, 319, 60, 60, 37, 60, - /* 120 */ 60, 60, 19, 304, 206, 60, 62, 60, 60, 60, - /* 130 */ 60, 306, 60, 60, 60, 202, 87, 289, 212, 60, - /* 140 */ 60, 81, 60, 60, 60, 71, 303, 61, 291, 295, - /* 150 */ 342, 205, 177, 60, 37, 60, 60, 60, 60, 306, - /* 160 */ 60, 60, 60, 32, 87, 289, 23, 60, 60, 88, - /* 170 */ 60, 60, 60, 214, 302, 40, 60, 81, 60, 60, - /* 180 */ 60, 60, 178, 60, 60, 60, 327, 87, 289, 327, - /* 190 */ 60, 60, 64, 60, 60, 60, 38, 236, 237, 326, - /* 200 */ 325, 9, 8, 325, 60, 88, 60, 60, 381, 381, - /* 210 */ 307, 381, 381, 381, 179, 87, 289, 14, 381, 381, - /* 220 */ 67, 381, 381, 381, 282, 28, 334, 60, 64, 60, - /* 230 */ 60, 380, 380, 59, 380, 380, 380, 29, 87, 289, - /* 240 */ 343, 380, 380, 91, 380, 380, 380, 282, 282, 333, - /* 250 */ 68, 282, 23, 338, 282, 381, 337, 381, 381, 65, - /* 260 */ 65, 40, 65, 65, 65, 86, 87, 289, 27, 65, - /* 270 */ 65, 30, 65, 65, 65, 282, 85, 69, 380, 48, - /* 280 */ 380, 380, 66, 66, 171, 66, 66, 66, 284, 87, - /* 290 */ 289, 283, 66, 66, 84, 66, 66, 66, 21, 20, - /* 300 */ 239, 240, 25, 7, 324, 286, 65, 52, 65, 65, - /* 310 */ 377, 377, 26, 377, 377, 377, 178, 87, 289, 312, - /* 320 */ 377, 377, 56, 377, 377, 377, 55, 57, 58, 66, - /* 330 */ 213, 66, 66, 376, 376, 3, 376, 376, 376, 20, - /* 340 */ 87, 289, 200, 376, 376, 199, 376, 376, 376, 43, - /* 350 */ 17, 16, 285, 36, 263, 230, 24, 377, 13, 12, - /* 360 */ 377, 4, 219, 250, 171, 172, 245, 246, 247, 70, - /* 370 */ 249, 6, 2, 279, 54, 276, 89, 278, 90, 82, - /* 380 */ 376, 277, 277, 376, 277, 277, 277, 277, 19, 277, - /* 390 */ 13, 12, 207, 277, 277, 277, 171, 172, 245, 246, - /* 400 */ 247, 70, 249, 6, 277, 327, 277, 277, 327, 277, - /* 410 */ 277, 277, 76, 33, 277, 39, 277, 277, 39, 325, - /* 420 */ 19, 31, 325, 76, 33, 277, 78, 41, 300, 301, - /* 430 */ 277, 300, 31, 277, 277, 277, 277, 277, 35, 300, - /* 440 */ 301, 277, 300, 277, 76, 33, 277, 277, 277, 277, - /* 450 */ 277, 277, 277, 31, 277, 277, 277, 277, 277, 45, - /* 460 */ 300, 301, 277, 300, + /* 0 */ 392, 390, 94, 494, 20, 67, 393, 390, 27, 25, + /* 10 */ 23, 22, 21, 400, 390, 321, 52, 131, 414, 144, + /* 20 */ 492, 74, 46, 401, 266, 403, 19, 89, 98, 119, + /* 30 */ 284, 285, 286, 287, 288, 289, 290, 292, 293, 294, + /* 40 */ 27, 25, 23, 22, 21, 23, 22, 21, 346, 465, + /* 50 */ 463, 247, 248, 249, 250, 145, 253, 320, 19, 89, + /* 60 */ 98, 132, 284, 285, 286, 287, 288, 289, 290, 292, + /* 70 */ 293, 294, 26, 24, 110, 344, 345, 347, 348, 247, + /* 80 */ 248, 249, 250, 145, 253, 260, 103, 6, 400, 390, + /* 90 */ 494, 8, 143, 414, 144, 3, 317, 41, 401, 111, + /* 100 */ 403, 439, 36, 53, 70, 96, 435, 492, 400, 390, + /* 110 */ 143, 414, 143, 414, 144, 469, 114, 41, 401, 128, + /* 120 */ 403, 439, 450, 148, 54, 96, 435, 450, 143, 414, + /* 130 */ 494, 34, 63, 36, 118, 490, 400, 390, 137, 448, + /* 140 */ 143, 414, 144, 493, 447, 41, 401, 492, 403, 439, + /* 150 */ 26, 24, 322, 96, 435, 54, 17, 247, 248, 249, + /* 160 */ 250, 145, 253, 454, 103, 28, 291, 400, 390, 295, + /* 170 */ 136, 143, 414, 144, 5, 4, 41, 401, 329, 403, + /* 180 */ 439, 138, 400, 390, 438, 435, 143, 414, 144, 415, + /* 190 */ 258, 41, 401, 76, 403, 439, 120, 115, 113, 123, + /* 200 */ 435, 400, 390, 377, 378, 143, 414, 144, 78, 34, + /* 210 */ 40, 401, 42, 403, 439, 343, 400, 390, 88, 435, + /* 220 */ 131, 414, 144, 26, 24, 46, 401, 7, 403, 134, + /* 230 */ 247, 248, 249, 250, 145, 253, 112, 103, 6, 476, + /* 240 */ 27, 25, 23, 22, 21, 109, 72, 26, 24, 130, + /* 250 */ 31, 140, 104, 462, 247, 248, 249, 250, 145, 253, + /* 260 */ 450, 103, 28, 400, 390, 55, 253, 143, 414, 144, + /* 270 */ 107, 57, 41, 401, 60, 403, 439, 446, 124, 108, + /* 280 */ 281, 436, 34, 400, 390, 475, 95, 143, 414, 144, + /* 290 */ 5, 4, 47, 401, 34, 403, 141, 59, 400, 390, + /* 300 */ 374, 375, 143, 414, 144, 2, 34, 86, 401, 105, + /* 310 */ 403, 400, 390, 302, 456, 143, 414, 144, 16, 122, + /* 320 */ 86, 401, 121, 403, 27, 25, 23, 22, 21, 133, + /* 330 */ 508, 400, 390, 62, 106, 143, 414, 144, 49, 1, + /* 340 */ 86, 401, 97, 403, 400, 390, 64, 317, 143, 414, + /* 350 */ 144, 13, 29, 47, 401, 296, 403, 400, 390, 421, + /* 360 */ 257, 143, 414, 144, 44, 260, 86, 401, 102, 403, + /* 370 */ 451, 400, 390, 30, 65, 143, 414, 144, 261, 29, + /* 380 */ 83, 401, 264, 403, 400, 390, 510, 466, 143, 414, + /* 390 */ 144, 509, 139, 80, 401, 99, 403, 400, 390, 135, + /* 400 */ 142, 143, 414, 144, 258, 77, 84, 401, 397, 403, + /* 410 */ 395, 400, 390, 75, 491, 143, 414, 144, 10, 29, + /* 420 */ 81, 401, 11, 403, 400, 390, 35, 56, 143, 414, + /* 430 */ 144, 340, 58, 85, 401, 342, 403, 400, 390, 336, + /* 440 */ 48, 143, 414, 144, 61, 38, 411, 401, 335, 403, + /* 450 */ 116, 400, 390, 117, 39, 143, 414, 144, 395, 12, + /* 460 */ 410, 401, 4, 403, 400, 390, 282, 315, 143, 414, + /* 470 */ 144, 314, 32, 409, 401, 69, 403, 400, 390, 33, + /* 480 */ 394, 143, 414, 144, 51, 368, 92, 401, 14, 403, + /* 490 */ 9, 400, 390, 37, 357, 143, 414, 144, 363, 79, + /* 500 */ 91, 401, 362, 403, 400, 390, 100, 367, 143, 414, + /* 510 */ 144, 366, 101, 93, 401, 251, 403, 400, 390, 384, + /* 520 */ 383, 143, 414, 144, 15, 147, 90, 401, 382, 403, + /* 530 */ 382, 400, 390, 382, 382, 143, 414, 144, 382, 382, + /* 540 */ 82, 401, 382, 403, 400, 390, 382, 382, 143, 414, + /* 550 */ 144, 382, 382, 87, 401, 382, 403, 127, 45, 382, + /* 560 */ 382, 382, 127, 45, 382, 382, 43, 382, 382, 382, + /* 570 */ 382, 43, 382, 382, 129, 66, 444, 445, 382, 444, + /* 580 */ 68, 444, 126, 382, 125, 382, 382, 127, 45, 382, + /* 590 */ 382, 382, 127, 45, 382, 382, 43, 382, 382, 382, + /* 600 */ 382, 43, 382, 381, 146, 50, 444, 445, 382, 444, + /* 610 */ 71, 444, 445, 382, 444, 27, 25, 23, 22, 21, + /* 620 */ 27, 25, 23, 22, 21, 382, 382, 382, 382, 261, + /* 630 */ 382, 382, 18, 494, 382, 382, 266, 382, 27, 25, + /* 640 */ 23, 22, 21, 382, 382, 382, 53, 382, 73, 382, + /* 650 */ 492, 27, 25, 23, 22, 21, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 47, 48, 73, 50, 51, 52, 55, 54, 55, 92, - /* 10 */ 57, 58, 59, 60, 61, 62, 15, 16, 17, 18, - /* 20 */ 19, 20, 21, 22, 1, 2, 53, 54, 64, 65, - /* 30 */ 77, 9, 10, 11, 12, 62, 83, 84, 3, 22, - /* 40 */ 39, 68, 69, 70, 86, 72, 23, 94, 95, 96, - /* 50 */ 97, 47, 48, 20, 50, 51, 52, 22, 54, 55, - /* 60 */ 27, 57, 58, 59, 60, 61, 62, 29, 30, 63, - /* 70 */ 64, 65, 63, 64, 65, 63, 64, 65, 47, 48, - /* 80 */ 74, 50, 51, 52, 23, 54, 55, 26, 57, 58, - /* 90 */ 59, 60, 61, 62, 90, 91, 9, 10, 94, 95, - /* 100 */ 96, 97, 15, 20, 86, 22, 47, 48, 56, 50, - /* 110 */ 51, 52, 76, 54, 55, 84, 57, 58, 59, 60, - /* 120 */ 61, 62, 39, 71, 5, 94, 95, 96, 97, 47, - /* 130 */ 48, 56, 50, 51, 52, 30, 54, 55, 20, 57, - /* 140 */ 58, 59, 60, 61, 62, 27, 71, 63, 64, 65, - /* 150 */ 91, 32, 20, 94, 95, 96, 97, 47, 48, 56, - /* 160 */ 50, 51, 52, 54, 54, 55, 22, 57, 58, 59, - /* 170 */ 60, 61, 62, 11, 71, 31, 94, 95, 96, 97, - /* 180 */ 47, 48, 20, 50, 51, 52, 75, 54, 55, 75, - /* 190 */ 57, 58, 59, 60, 61, 62, 85, 40, 41, 85, - /* 200 */ 89, 11, 12, 89, 94, 95, 96, 97, 47, 48, - /* 210 */ 56, 50, 51, 52, 20, 54, 55, 25, 57, 58, - /* 220 */ 42, 60, 61, 62, 47, 26, 49, 94, 95, 96, - /* 230 */ 97, 47, 48, 66, 50, 51, 52, 38, 54, 55, - /* 240 */ 93, 57, 58, 13, 60, 61, 62, 47, 47, 49, - /* 250 */ 49, 47, 22, 49, 47, 94, 49, 96, 97, 47, - /* 260 */ 48, 31, 50, 51, 52, 4, 54, 55, 35, 57, - /* 270 */ 58, 37, 60, 61, 62, 47, 87, 49, 94, 88, - /* 280 */ 96, 97, 47, 48, 15, 50, 51, 52, 47, 54, - /* 290 */ 55, 47, 57, 58, 33, 60, 61, 62, 1, 2, - /* 300 */ 43, 44, 26, 22, 88, 24, 94, 87, 96, 97, - /* 310 */ 47, 48, 36, 50, 51, 52, 20, 54, 55, 82, - /* 320 */ 57, 58, 80, 60, 61, 62, 81, 79, 78, 94, - /* 330 */ 20, 96, 97, 47, 48, 26, 50, 51, 52, 2, - /* 340 */ 54, 55, 23, 57, 58, 23, 60, 61, 62, 24, - /* 350 */ 26, 28, 24, 24, 23, 23, 5, 94, 9, 10, - /* 360 */ 97, 26, 23, 23, 15, 16, 17, 18, 19, 20, - /* 370 */ 21, 22, 34, 0, 24, 45, 46, 0, 14, 46, - /* 380 */ 94, 98, 98, 97, 98, 98, 98, 98, 39, 98, - /* 390 */ 9, 10, 11, 98, 98, 98, 15, 16, 17, 18, - /* 400 */ 19, 20, 21, 22, 98, 75, 98, 98, 75, 98, - /* 410 */ 98, 98, 53, 54, 98, 85, 98, 98, 85, 89, - /* 420 */ 39, 62, 89, 53, 54, 98, 67, 68, 69, 70, - /* 430 */ 98, 72, 62, 98, 98, 98, 98, 98, 68, 69, - /* 440 */ 70, 98, 72, 98, 53, 54, 98, 98, 98, 98, - /* 450 */ 98, 98, 98, 62, 98, 98, 98, 98, 98, 68, - /* 460 */ 69, 70, 98, 72, + /* 0 */ 75, 76, 77, 103, 89, 90, 75, 76, 8, 9, + /* 10 */ 10, 11, 12, 75, 76, 4, 116, 79, 80, 81, + /* 20 */ 120, 123, 84, 85, 24, 87, 26, 27, 28, 22, + /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + /* 40 */ 8, 9, 10, 11, 12, 10, 11, 12, 29, 83, + /* 50 */ 112, 15, 16, 17, 18, 19, 20, 46, 26, 27, + /* 60 */ 28, 74, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 8, 9, 55, 56, 57, 58, 59, 15, + /* 80 */ 16, 17, 18, 19, 20, 22, 22, 23, 75, 76, + /* 90 */ 103, 27, 79, 80, 81, 43, 44, 84, 85, 115, + /* 100 */ 87, 88, 23, 116, 41, 92, 93, 120, 75, 76, + /* 110 */ 79, 80, 79, 80, 81, 102, 85, 84, 85, 101, + /* 120 */ 87, 88, 82, 13, 45, 92, 93, 82, 79, 80, + /* 130 */ 103, 67, 108, 23, 85, 102, 75, 76, 21, 99, + /* 140 */ 79, 80, 81, 116, 99, 84, 85, 120, 87, 88, + /* 150 */ 8, 9, 10, 92, 93, 45, 26, 15, 16, 17, + /* 160 */ 18, 19, 20, 102, 22, 23, 36, 75, 76, 39, + /* 170 */ 3, 79, 80, 81, 1, 2, 84, 85, 10, 87, + /* 180 */ 88, 64, 75, 76, 92, 93, 79, 80, 81, 80, + /* 190 */ 22, 84, 85, 117, 87, 88, 50, 51, 52, 92, + /* 200 */ 93, 75, 76, 71, 72, 79, 80, 81, 117, 67, + /* 210 */ 84, 85, 21, 87, 88, 24, 75, 76, 92, 93, + /* 220 */ 79, 80, 81, 8, 9, 84, 85, 104, 87, 62, + /* 230 */ 15, 16, 17, 18, 19, 20, 54, 22, 23, 114, + /* 240 */ 8, 9, 10, 11, 12, 53, 105, 8, 9, 22, + /* 250 */ 23, 21, 111, 112, 15, 16, 17, 18, 19, 20, + /* 260 */ 82, 22, 23, 75, 76, 113, 20, 79, 80, 81, + /* 270 */ 76, 21, 84, 85, 24, 87, 88, 99, 27, 76, + /* 280 */ 29, 93, 67, 75, 76, 114, 76, 79, 80, 81, + /* 290 */ 1, 2, 84, 85, 67, 87, 66, 113, 75, 76, + /* 300 */ 68, 69, 79, 80, 81, 61, 67, 84, 85, 86, + /* 310 */ 87, 75, 76, 24, 110, 79, 80, 81, 2, 60, + /* 320 */ 84, 85, 86, 87, 8, 9, 10, 11, 12, 121, + /* 330 */ 122, 75, 76, 109, 48, 79, 80, 81, 107, 47, + /* 340 */ 84, 85, 86, 87, 75, 76, 106, 44, 79, 80, + /* 350 */ 81, 23, 21, 84, 85, 24, 87, 75, 76, 91, + /* 360 */ 22, 79, 80, 81, 79, 22, 84, 85, 86, 87, + /* 370 */ 82, 75, 76, 40, 94, 79, 80, 81, 22, 21, + /* 380 */ 84, 85, 24, 87, 75, 76, 124, 83, 79, 80, + /* 390 */ 81, 122, 63, 84, 85, 70, 87, 75, 76, 118, + /* 400 */ 65, 79, 80, 81, 22, 118, 84, 85, 23, 87, + /* 410 */ 25, 75, 76, 119, 119, 79, 80, 81, 21, 21, + /* 420 */ 84, 85, 49, 87, 75, 76, 21, 24, 79, 80, + /* 430 */ 81, 24, 23, 84, 85, 24, 87, 75, 76, 24, + /* 440 */ 23, 79, 80, 81, 23, 23, 84, 85, 24, 87, + /* 450 */ 15, 75, 76, 21, 23, 79, 80, 81, 25, 49, + /* 460 */ 84, 85, 2, 87, 75, 76, 29, 24, 79, 80, + /* 470 */ 81, 24, 42, 84, 85, 25, 87, 75, 76, 21, + /* 480 */ 25, 79, 80, 81, 25, 24, 84, 85, 21, 87, + /* 490 */ 49, 75, 76, 4, 24, 79, 80, 81, 15, 25, + /* 500 */ 84, 85, 15, 87, 75, 76, 15, 15, 79, 80, + /* 510 */ 81, 15, 15, 84, 85, 17, 87, 75, 76, 0, + /* 520 */ 0, 79, 80, 81, 23, 14, 84, 85, 125, 87, + /* 530 */ 125, 75, 76, 125, 125, 79, 80, 81, 125, 125, + /* 540 */ 84, 85, 125, 87, 75, 76, 125, 125, 79, 80, + /* 550 */ 81, 125, 125, 84, 85, 125, 87, 78, 79, 125, + /* 560 */ 125, 125, 78, 79, 125, 125, 87, 125, 125, 125, + /* 570 */ 125, 87, 125, 125, 95, 96, 97, 98, 125, 100, + /* 580 */ 96, 97, 98, 125, 100, 125, 125, 78, 79, 125, + /* 590 */ 125, 125, 78, 79, 125, 125, 87, 125, 125, 125, + /* 600 */ 125, 87, 125, 73, 74, 96, 97, 98, 125, 100, + /* 610 */ 96, 97, 98, 125, 100, 8, 9, 10, 11, 12, + /* 620 */ 8, 9, 10, 11, 12, 125, 125, 125, 125, 22, + /* 630 */ 125, 125, 2, 103, 125, 125, 24, 125, 8, 9, + /* 640 */ 10, 11, 12, 125, 125, 125, 116, 125, 41, 125, + /* 650 */ 120, 8, 9, 10, 11, 12, 125, 125, 125, 125, + /* 660 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 670 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 680 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 690 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 700 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 710 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 720 */ 125, 125, 125, 125, 125, }; -#define YY_SHIFT_COUNT (91) +#define YY_SHIFT_COUNT (148) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (381) +#define YY_SHIFT_MAX (643) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 230, 381, 349, 349, 349, 349, 349, 349, 349, 349, - /* 10 */ 349, 349, 1, 1, 83, 83, 83, 83, 35, 144, - /* 20 */ 35, 35, 35, 144, 144, 87, 87, 87, 87, 87, - /* 30 */ 87, 33, 33, 33, 17, 38, 162, 157, 261, 261, - /* 40 */ 119, 105, 105, 132, 194, 105, 192, 178, 233, 234, - /* 50 */ 269, 269, 234, 233, 296, 464, 464, 464, 464, 464, - /* 60 */ 22, 23, 118, 297, 61, 190, 190, 257, 276, 199, - /* 70 */ 281, 310, 309, 337, 319, 322, 325, 323, 324, 328, - /* 80 */ 329, 331, 332, 335, 338, 339, 351, 350, 340, 373, - /* 90 */ 377, 364, + /* 0 */ 110, 64, 64, 64, 64, 64, 64, 142, 215, 239, + /* 10 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 20 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 30 */ 227, 227, 227, 227, 79, 36, 79, 79, 7, 7, + /* 40 */ 0, 32, 36, 63, 63, 63, 607, 232, 19, 146, + /* 50 */ 52, 168, 167, 167, 11, 182, 192, 246, 246, 182, + /* 60 */ 192, 246, 244, 259, 286, 292, 303, 328, 303, 338, + /* 70 */ 343, 303, 333, 356, 325, 329, 335, 335, 329, 382, + /* 80 */ 316, 630, 612, 643, 643, 643, 643, 643, 289, 130, + /* 90 */ 35, 35, 35, 35, 191, 250, 173, 331, 251, 132, + /* 100 */ 117, 230, 358, 385, 397, 398, 373, 403, 407, 409, + /* 110 */ 405, 411, 417, 421, 415, 422, 424, 435, 432, 433, + /* 120 */ 431, 398, 410, 460, 437, 443, 447, 450, 430, 458, + /* 130 */ 455, 459, 461, 467, 441, 470, 489, 483, 487, 491, + /* 140 */ 492, 496, 497, 474, 501, 498, 519, 520, 511, }; -#define YY_REDUCE_COUNT (59) -#define YY_REDUCE_MIN (-83) -#define YY_REDUCE_MAX (391) +#define YY_REDUCE_COUNT (79) +#define YY_REDUCE_MIN (-102) +#define YY_REDUCE_MAX (530) static const short yy_reduce_ofst[] = { - /* 0 */ 330, -47, 4, 31, 59, 82, 110, 133, 161, 184, - /* 10 */ 212, 235, 263, 286, 359, -27, 370, 391, 6, 333, - /* 20 */ 9, 12, 84, 111, 114, 177, 200, 201, 204, 207, - /* 30 */ 228, 52, 75, 103, -36, -71, -49, -83, -42, 18, - /* 40 */ 36, -71, -71, 109, 154, -71, 167, 147, 189, 191, - /* 50 */ 241, 244, 216, 220, -49, 237, 245, 242, 248, 250, + /* 0 */ 530, 13, 33, 61, 92, 107, 126, 141, 188, 208, + /* 10 */ -62, 223, 236, 256, 269, 282, 296, 309, 322, 336, + /* 20 */ 349, 362, 376, 389, 402, 416, 429, 442, 456, 469, + /* 30 */ 479, 484, 509, 514, -13, -75, -100, 27, 31, 49, + /* 40 */ -85, -85, -69, 40, 45, 178, -34, -102, -16, 24, + /* 50 */ 18, 109, 76, 91, 123, 125, 152, 194, 203, 171, + /* 60 */ 184, 210, 204, 224, 231, 240, 18, 268, 18, 285, + /* 70 */ 288, 18, 280, 304, 262, 281, 294, 295, 287, 109, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 10 */ 275, 275, 275, 275, 275, 275, 275, 275, 365, 275, - /* 20 */ 365, 365, 365, 275, 275, 275, 275, 275, 275, 275, - /* 30 */ 275, 305, 305, 305, 365, 275, 275, 344, 329, 329, - /* 40 */ 313, 298, 275, 275, 275, 299, 275, 347, 331, 335, - /* 50 */ 275, 275, 335, 331, 275, 370, 369, 368, 367, 366, - /* 60 */ 374, 275, 320, 340, 275, 379, 378, 275, 332, 336, - /* 70 */ 287, 275, 317, 293, 364, 301, 275, 275, 297, 286, - /* 80 */ 275, 275, 275, 330, 275, 275, 275, 275, 275, 275, - /* 90 */ 275, 275, + /* 0 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 10 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 20 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 30 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 40 */ 380, 380, 380, 449, 449, 449, 464, 511, 380, 472, + /* 50 */ 380, 380, 496, 496, 457, 479, 477, 380, 380, 479, + /* 60 */ 477, 380, 489, 487, 470, 468, 442, 380, 380, 380, + /* 70 */ 380, 443, 380, 380, 514, 498, 502, 502, 498, 380, + /* 80 */ 380, 380, 380, 418, 417, 416, 412, 413, 380, 380, + /* 90 */ 407, 408, 406, 405, 380, 380, 507, 380, 380, 380, + /* 100 */ 499, 503, 380, 396, 461, 471, 380, 380, 380, 380, + /* 110 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 396, + /* 120 */ 380, 488, 380, 437, 380, 517, 445, 380, 380, 441, + /* 130 */ 395, 380, 380, 497, 380, 380, 380, 380, 380, 380, + /* 140 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, }; /********** End of lemon-generated parsing tables *****************************/ @@ -451,101 +517,128 @@ static const char *const yyTokenName[] = { /* 0 */ "$", /* 1 */ "OR", /* 2 */ "AND", - /* 3 */ "NOT", - /* 4 */ "UNION", - /* 5 */ "ALL", - /* 6 */ "MINUS", - /* 7 */ "EXCEPT", - /* 8 */ "INTERSECT", - /* 9 */ "NK_PLUS", - /* 10 */ "NK_MINUS", - /* 11 */ "NK_STAR", - /* 12 */ "NK_SLASH", + /* 3 */ "UNION", + /* 4 */ "ALL", + /* 5 */ "MINUS", + /* 6 */ "EXCEPT", + /* 7 */ "INTERSECT", + /* 8 */ "NK_PLUS", + /* 9 */ "NK_MINUS", + /* 10 */ "NK_STAR", + /* 11 */ "NK_SLASH", + /* 12 */ "NK_REM", /* 13 */ "SHOW", /* 14 */ "DATABASES", /* 15 */ "NK_INTEGER", /* 16 */ "NK_FLOAT", /* 17 */ "NK_STRING", /* 18 */ "NK_BOOL", - /* 19 */ "NK_NOW", - /* 20 */ "NK_ID", - /* 21 */ "NK_QUESTION", - /* 22 */ "NK_LP", - /* 23 */ "NK_RP", - /* 24 */ "NK_DOT", - /* 25 */ "FROM", - /* 26 */ "NK_COMMA", - /* 27 */ "AS", - /* 28 */ "JOIN", - /* 29 */ "ON", - /* 30 */ "INNER", - /* 31 */ "SELECT", - /* 32 */ "DISTINCT", - /* 33 */ "ORDER", - /* 34 */ "BY", - /* 35 */ "SLIMIT", - /* 36 */ "SOFFSET", - /* 37 */ "LIMIT", - /* 38 */ "OFFSET", - /* 39 */ "NK_LR", - /* 40 */ "ASC", - /* 41 */ "DESC", - /* 42 */ "NULLS", - /* 43 */ "FIRST", - /* 44 */ "LAST", - /* 45 */ "cmd", - /* 46 */ "query_expression", - /* 47 */ "unsigned_integer", - /* 48 */ "unsigned_approximate_numeric", - /* 49 */ "signed_integer", - /* 50 */ "unsigned_literal", - /* 51 */ "unsigned_numeric_literal", - /* 52 */ "general_literal", - /* 53 */ "db_name", - /* 54 */ "table_name", - /* 55 */ "column_name", - /* 56 */ "table_alias", - /* 57 */ "unsigned_value_specification", - /* 58 */ "value_expression_primary", - /* 59 */ "value_expression", - /* 60 */ "nonparenthesized_value_expression_primary", - /* 61 */ "column_reference", - /* 62 */ "subquery", - /* 63 */ "boolean_value_expression", - /* 64 */ "boolean_primary", - /* 65 */ "predicate", - /* 66 */ "from_clause", - /* 67 */ "table_reference_list", - /* 68 */ "table_reference", - /* 69 */ "table_primary", - /* 70 */ "joined_table", - /* 71 */ "correlation_or_recognition_opt", - /* 72 */ "parenthesized_joined_table", - /* 73 */ "join_type", - /* 74 */ "search_condition", - /* 75 */ "query_specification", - /* 76 */ "set_quantifier_opt", - /* 77 */ "select_list", - /* 78 */ "where_clause_opt", - /* 79 */ "partition_by_clause_opt", - /* 80 */ "twindow_clause_opt", - /* 81 */ "group_by_clause_opt", - /* 82 */ "having_clause_opt", - /* 83 */ "select_sublist", - /* 84 */ "select_item", - /* 85 */ "query_expression_body", - /* 86 */ "order_by_clause_opt", - /* 87 */ "slimit_clause_opt", - /* 88 */ "limit_clause_opt", - /* 89 */ "query_primary", - /* 90 */ "sort_specification_list", - /* 91 */ "sort_specification", - /* 92 */ "ordering_specification_opt", - /* 93 */ "null_ordering_opt", - /* 94 */ "value_function", - /* 95 */ "common_value_expression", - /* 96 */ "numeric_value_expression", - /* 97 */ "numeric_primary", + /* 19 */ "TIMESTAMP", + /* 20 */ "NK_VARIABLE", + /* 21 */ "NK_COMMA", + /* 22 */ "NK_ID", + /* 23 */ "NK_LP", + /* 24 */ "NK_RP", + /* 25 */ "NK_DOT", + /* 26 */ "BETWEEN", + /* 27 */ "NOT", + /* 28 */ "IS", + /* 29 */ "NULL", + /* 30 */ "NK_LT", + /* 31 */ "NK_GT", + /* 32 */ "NK_LE", + /* 33 */ "NK_GE", + /* 34 */ "NK_NE", + /* 35 */ "NK_EQ", + /* 36 */ "LIKE", + /* 37 */ "MATCH", + /* 38 */ "NMATCH", + /* 39 */ "IN", + /* 40 */ "FROM", + /* 41 */ "AS", + /* 42 */ "JOIN", + /* 43 */ "ON", + /* 44 */ "INNER", + /* 45 */ "SELECT", + /* 46 */ "DISTINCT", + /* 47 */ "WHERE", + /* 48 */ "PARTITION", + /* 49 */ "BY", + /* 50 */ "SESSION", + /* 51 */ "STATE_WINDOW", + /* 52 */ "INTERVAL", + /* 53 */ "SLIDING", + /* 54 */ "FILL", + /* 55 */ "VALUE", + /* 56 */ "NONE", + /* 57 */ "PREV", + /* 58 */ "LINEAR", + /* 59 */ "NEXT", + /* 60 */ "GROUP", + /* 61 */ "HAVING", + /* 62 */ "ORDER", + /* 63 */ "SLIMIT", + /* 64 */ "SOFFSET", + /* 65 */ "LIMIT", + /* 66 */ "OFFSET", + /* 67 */ "NK_LR", + /* 68 */ "ASC", + /* 69 */ "DESC", + /* 70 */ "NULLS", + /* 71 */ "FIRST", + /* 72 */ "LAST", + /* 73 */ "cmd", + /* 74 */ "query_expression", + /* 75 */ "literal", + /* 76 */ "duration_literal", + /* 77 */ "literal_list", + /* 78 */ "db_name", + /* 79 */ "table_name", + /* 80 */ "column_name", + /* 81 */ "function_name", + /* 82 */ "table_alias", + /* 83 */ "column_alias", + /* 84 */ "expression", + /* 85 */ "column_reference", + /* 86 */ "expression_list", + /* 87 */ "subquery", + /* 88 */ "predicate", + /* 89 */ "compare_op", + /* 90 */ "in_op", + /* 91 */ "in_predicate_value", + /* 92 */ "boolean_value_expression", + /* 93 */ "boolean_primary", + /* 94 */ "from_clause", + /* 95 */ "table_reference_list", + /* 96 */ "table_reference", + /* 97 */ "table_primary", + /* 98 */ "joined_table", + /* 99 */ "alias_opt", + /* 100 */ "parenthesized_joined_table", + /* 101 */ "join_type", + /* 102 */ "search_condition", + /* 103 */ "query_specification", + /* 104 */ "set_quantifier_opt", + /* 105 */ "select_list", + /* 106 */ "where_clause_opt", + /* 107 */ "partition_by_clause_opt", + /* 108 */ "twindow_clause_opt", + /* 109 */ "group_by_clause_opt", + /* 110 */ "having_clause_opt", + /* 111 */ "select_sublist", + /* 112 */ "select_item", + /* 113 */ "sliding_opt", + /* 114 */ "fill_opt", + /* 115 */ "fill_mode", + /* 116 */ "query_expression_body", + /* 117 */ "order_by_clause_opt", + /* 118 */ "slimit_clause_opt", + /* 119 */ "limit_clause_opt", + /* 120 */ "query_primary", + /* 121 */ "sort_specification_list", + /* 122 */ "sort_specification", + /* 123 */ "ordering_specification_opt", + /* 124 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -555,110 +648,139 @@ static const char *const yyTokenName[] = { static const char *const yyRuleName[] = { /* 0 */ "cmd ::= SHOW DATABASES", /* 1 */ "cmd ::= query_expression", - /* 2 */ "unsigned_integer ::= NK_INTEGER", - /* 3 */ "unsigned_approximate_numeric ::= NK_FLOAT", - /* 4 */ "signed_integer ::= unsigned_integer", - /* 5 */ "signed_integer ::= NK_PLUS unsigned_integer", - /* 6 */ "signed_integer ::= NK_MINUS unsigned_integer", - /* 7 */ "db_name ::= NK_ID", - /* 8 */ "table_name ::= NK_ID", - /* 9 */ "column_name ::= NK_ID", - /* 10 */ "table_alias ::= NK_ID", - /* 11 */ "column_reference ::= column_name", - /* 12 */ "column_reference ::= table_name NK_DOT column_name", - /* 13 */ "boolean_value_expression ::= boolean_primary", - /* 14 */ "boolean_value_expression ::= NOT boolean_primary", - /* 15 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 16 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 17 */ "boolean_primary ::= predicate", - /* 18 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 19 */ "from_clause ::= FROM table_reference_list", - /* 20 */ "table_reference_list ::= table_reference", - /* 21 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 22 */ "table_reference ::= table_primary", - /* 23 */ "table_reference ::= joined_table", - /* 24 */ "table_primary ::= table_name correlation_or_recognition_opt", - /* 25 */ "table_primary ::= db_name NK_DOT table_name correlation_or_recognition_opt", - /* 26 */ "table_primary ::= subquery correlation_or_recognition_opt", - /* 27 */ "correlation_or_recognition_opt ::=", - /* 28 */ "correlation_or_recognition_opt ::= table_alias", - /* 29 */ "correlation_or_recognition_opt ::= AS table_alias", - /* 30 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 31 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 32 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 33 */ "join_type ::= INNER", - /* 34 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 35 */ "set_quantifier_opt ::=", - /* 36 */ "set_quantifier_opt ::= DISTINCT", - /* 37 */ "set_quantifier_opt ::= ALL", - /* 38 */ "select_list ::= NK_STAR", - /* 39 */ "select_list ::= select_sublist", - /* 40 */ "select_sublist ::= select_item", - /* 41 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 42 */ "select_item ::= value_expression", - /* 43 */ "select_item ::= value_expression NK_ID", - /* 44 */ "select_item ::= value_expression AS NK_ID", - /* 45 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 46 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 47 */ "query_expression_body ::= query_primary", - /* 48 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 49 */ "query_primary ::= query_specification", - /* 50 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", - /* 51 */ "order_by_clause_opt ::=", - /* 52 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 53 */ "slimit_clause_opt ::=", - /* 54 */ "slimit_clause_opt ::= SLIMIT signed_integer", - /* 55 */ "slimit_clause_opt ::= SLIMIT signed_integer SOFFSET signed_integer", - /* 56 */ "slimit_clause_opt ::= SLIMIT signed_integer NK_COMMA signed_integer", - /* 57 */ "limit_clause_opt ::=", - /* 58 */ "limit_clause_opt ::= LIMIT signed_integer", - /* 59 */ "limit_clause_opt ::= LIMIT signed_integer OFFSET signed_integer", - /* 60 */ "limit_clause_opt ::= LIMIT signed_integer NK_COMMA signed_integer", - /* 61 */ "subquery ::= NK_LR query_expression NK_RP", - /* 62 */ "search_condition ::= boolean_value_expression", - /* 63 */ "sort_specification_list ::= sort_specification", - /* 64 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 65 */ "sort_specification ::= value_expression ordering_specification_opt null_ordering_opt", - /* 66 */ "ordering_specification_opt ::=", - /* 67 */ "ordering_specification_opt ::= ASC", - /* 68 */ "ordering_specification_opt ::= DESC", - /* 69 */ "null_ordering_opt ::=", - /* 70 */ "null_ordering_opt ::= NULLS FIRST", - /* 71 */ "null_ordering_opt ::= NULLS LAST", - /* 72 */ "unsigned_literal ::= unsigned_numeric_literal", - /* 73 */ "unsigned_literal ::= general_literal", - /* 74 */ "unsigned_numeric_literal ::= unsigned_integer", - /* 75 */ "unsigned_numeric_literal ::= unsigned_approximate_numeric", - /* 76 */ "general_literal ::= NK_STRING", - /* 77 */ "general_literal ::= NK_BOOL", - /* 78 */ "general_literal ::= NK_NOW", - /* 79 */ "unsigned_value_specification ::= unsigned_literal", - /* 80 */ "unsigned_value_specification ::= NK_QUESTION", - /* 81 */ "value_expression_primary ::= NK_LP value_expression NK_RP", - /* 82 */ "value_expression_primary ::= nonparenthesized_value_expression_primary", - /* 83 */ "nonparenthesized_value_expression_primary ::= unsigned_value_specification", - /* 84 */ "nonparenthesized_value_expression_primary ::= column_reference", - /* 85 */ "nonparenthesized_value_expression_primary ::= subquery", - /* 86 */ "table_primary ::= parenthesized_joined_table", - /* 87 */ "predicate ::=", - /* 88 */ "where_clause_opt ::=", - /* 89 */ "partition_by_clause_opt ::=", - /* 90 */ "twindow_clause_opt ::=", - /* 91 */ "group_by_clause_opt ::=", - /* 92 */ "having_clause_opt ::=", - /* 93 */ "value_function ::= NK_ID NK_LP value_expression NK_RP", - /* 94 */ "value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP", - /* 95 */ "value_expression ::= common_value_expression", - /* 96 */ "common_value_expression ::= numeric_value_expression", - /* 97 */ "numeric_value_expression ::= numeric_primary", - /* 98 */ "numeric_value_expression ::= NK_PLUS numeric_primary", - /* 99 */ "numeric_value_expression ::= NK_MINUS numeric_primary", - /* 100 */ "numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression", - /* 101 */ "numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression", - /* 102 */ "numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression", - /* 103 */ "numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression", - /* 104 */ "numeric_primary ::= value_expression_primary", - /* 105 */ "numeric_primary ::= value_function", + /* 2 */ "literal ::= NK_INTEGER", + /* 3 */ "literal ::= NK_FLOAT", + /* 4 */ "literal ::= NK_STRING", + /* 5 */ "literal ::= NK_BOOL", + /* 6 */ "literal ::= TIMESTAMP NK_STRING", + /* 7 */ "literal ::= duration_literal", + /* 8 */ "duration_literal ::= NK_VARIABLE", + /* 9 */ "literal_list ::= literal", + /* 10 */ "literal_list ::= literal_list NK_COMMA literal", + /* 11 */ "db_name ::= NK_ID", + /* 12 */ "table_name ::= NK_ID", + /* 13 */ "column_name ::= NK_ID", + /* 14 */ "function_name ::= NK_ID", + /* 15 */ "table_alias ::= NK_ID", + /* 16 */ "column_alias ::= NK_ID", + /* 17 */ "expression ::= literal", + /* 18 */ "expression ::= column_reference", + /* 19 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 20 */ "expression ::= subquery", + /* 21 */ "expression ::= NK_LP expression NK_RP", + /* 22 */ "expression ::= NK_PLUS expression", + /* 23 */ "expression ::= NK_MINUS expression", + /* 24 */ "expression ::= expression NK_PLUS expression", + /* 25 */ "expression ::= expression NK_MINUS expression", + /* 26 */ "expression ::= expression NK_STAR expression", + /* 27 */ "expression ::= expression NK_SLASH expression", + /* 28 */ "expression ::= expression NK_REM expression", + /* 29 */ "expression_list ::= expression", + /* 30 */ "expression_list ::= expression_list NK_COMMA expression", + /* 31 */ "column_reference ::= column_name", + /* 32 */ "column_reference ::= table_name NK_DOT column_name", + /* 33 */ "predicate ::= expression compare_op expression", + /* 34 */ "predicate ::= expression BETWEEN expression AND expression", + /* 35 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 36 */ "predicate ::= expression IS NULL", + /* 37 */ "predicate ::= expression IS NOT NULL", + /* 38 */ "predicate ::= expression in_op in_predicate_value", + /* 39 */ "compare_op ::= NK_LT", + /* 40 */ "compare_op ::= NK_GT", + /* 41 */ "compare_op ::= NK_LE", + /* 42 */ "compare_op ::= NK_GE", + /* 43 */ "compare_op ::= NK_NE", + /* 44 */ "compare_op ::= NK_EQ", + /* 45 */ "compare_op ::= LIKE", + /* 46 */ "compare_op ::= NOT LIKE", + /* 47 */ "compare_op ::= MATCH", + /* 48 */ "compare_op ::= NMATCH", + /* 49 */ "in_op ::= IN", + /* 50 */ "in_op ::= NOT IN", + /* 51 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 52 */ "boolean_value_expression ::= boolean_primary", + /* 53 */ "boolean_value_expression ::= NOT boolean_primary", + /* 54 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 55 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 56 */ "boolean_primary ::= predicate", + /* 57 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 58 */ "from_clause ::= FROM table_reference_list", + /* 59 */ "table_reference_list ::= table_reference", + /* 60 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 61 */ "table_reference ::= table_primary", + /* 62 */ "table_reference ::= joined_table", + /* 63 */ "table_primary ::= table_name alias_opt", + /* 64 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 65 */ "table_primary ::= subquery alias_opt", + /* 66 */ "alias_opt ::=", + /* 67 */ "alias_opt ::= table_alias", + /* 68 */ "alias_opt ::= AS table_alias", + /* 69 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 70 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 71 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 72 */ "join_type ::= INNER", + /* 73 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 74 */ "set_quantifier_opt ::=", + /* 75 */ "set_quantifier_opt ::= DISTINCT", + /* 76 */ "set_quantifier_opt ::= ALL", + /* 77 */ "select_list ::= NK_STAR", + /* 78 */ "select_list ::= select_sublist", + /* 79 */ "select_sublist ::= select_item", + /* 80 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 81 */ "select_item ::= expression", + /* 82 */ "select_item ::= expression column_alias", + /* 83 */ "select_item ::= expression AS column_alias", + /* 84 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 85 */ "where_clause_opt ::=", + /* 86 */ "where_clause_opt ::= WHERE search_condition", + /* 87 */ "partition_by_clause_opt ::=", + /* 88 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 89 */ "twindow_clause_opt ::=", + /* 90 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 91 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 92 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 93 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 94 */ "sliding_opt ::=", + /* 95 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 96 */ "fill_opt ::=", + /* 97 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 98 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 99 */ "fill_mode ::= NONE", + /* 100 */ "fill_mode ::= PREV", + /* 101 */ "fill_mode ::= NULL", + /* 102 */ "fill_mode ::= LINEAR", + /* 103 */ "fill_mode ::= NEXT", + /* 104 */ "group_by_clause_opt ::=", + /* 105 */ "group_by_clause_opt ::= GROUP BY expression_list", + /* 106 */ "having_clause_opt ::=", + /* 107 */ "having_clause_opt ::= HAVING search_condition", + /* 108 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 109 */ "query_expression_body ::= query_primary", + /* 110 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 111 */ "query_primary ::= query_specification", + /* 112 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", + /* 113 */ "order_by_clause_opt ::=", + /* 114 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 115 */ "slimit_clause_opt ::=", + /* 116 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 117 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 118 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 119 */ "limit_clause_opt ::=", + /* 120 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 121 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 122 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 123 */ "subquery ::= NK_LR query_expression NK_RP", + /* 124 */ "search_condition ::= boolean_value_expression", + /* 125 */ "sort_specification_list ::= sort_specification", + /* 126 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 127 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 128 */ "ordering_specification_opt ::=", + /* 129 */ "ordering_specification_opt ::= ASC", + /* 130 */ "ordering_specification_opt ::= DESC", + /* 131 */ "null_ordering_opt ::=", + /* 132 */ "null_ordering_opt ::= NULLS FIRST", + /* 133 */ "null_ordering_opt ::= NULLS LAST", + /* 134 */ "table_primary ::= parenthesized_joined_table", }; #endif /* NDEBUG */ @@ -785,83 +907,90 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 45: /* cmd */ - case 46: /* query_expression */ - case 47: /* unsigned_integer */ - case 48: /* unsigned_approximate_numeric */ - case 49: /* signed_integer */ - case 50: /* unsigned_literal */ - case 51: /* unsigned_numeric_literal */ - case 52: /* general_literal */ - case 57: /* unsigned_value_specification */ - case 58: /* value_expression_primary */ - case 59: /* value_expression */ - case 60: /* nonparenthesized_value_expression_primary */ - case 61: /* column_reference */ - case 62: /* subquery */ - case 63: /* boolean_value_expression */ - case 64: /* boolean_primary */ - case 65: /* predicate */ - case 66: /* from_clause */ - case 67: /* table_reference_list */ - case 68: /* table_reference */ - case 69: /* table_primary */ - case 70: /* joined_table */ - case 72: /* parenthesized_joined_table */ - case 74: /* search_condition */ - case 75: /* query_specification */ - case 78: /* where_clause_opt */ - case 80: /* twindow_clause_opt */ - case 82: /* having_clause_opt */ - case 84: /* select_item */ - case 85: /* query_expression_body */ - case 87: /* slimit_clause_opt */ - case 88: /* limit_clause_opt */ - case 89: /* query_primary */ - case 91: /* sort_specification */ - case 94: /* value_function */ - case 95: /* common_value_expression */ - case 96: /* numeric_value_expression */ - case 97: /* numeric_primary */ + case 73: /* cmd */ + case 74: /* query_expression */ + case 75: /* literal */ + case 76: /* duration_literal */ + case 84: /* expression */ + case 85: /* column_reference */ + case 87: /* subquery */ + case 88: /* predicate */ + case 91: /* in_predicate_value */ + case 92: /* boolean_value_expression */ + case 93: /* boolean_primary */ + case 94: /* from_clause */ + case 95: /* table_reference_list */ + case 96: /* table_reference */ + case 97: /* table_primary */ + case 98: /* joined_table */ + case 100: /* parenthesized_joined_table */ + case 102: /* search_condition */ + case 103: /* query_specification */ + case 106: /* where_clause_opt */ + case 108: /* twindow_clause_opt */ + case 110: /* having_clause_opt */ + case 112: /* select_item */ + case 113: /* sliding_opt */ + case 114: /* fill_opt */ + case 116: /* query_expression_body */ + case 118: /* slimit_clause_opt */ + case 119: /* limit_clause_opt */ + case 120: /* query_primary */ + case 122: /* sort_specification */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy168)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy212)); } break; - case 53: /* db_name */ - case 54: /* table_name */ - case 55: /* column_name */ - case 56: /* table_alias */ - case 71: /* correlation_or_recognition_opt */ + case 77: /* literal_list */ + case 86: /* expression_list */ + case 105: /* select_list */ + case 107: /* partition_by_clause_opt */ + case 109: /* group_by_clause_opt */ + case 111: /* select_sublist */ + case 117: /* order_by_clause_opt */ + case 121: /* sort_specification_list */ +{ + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy174)); +} + break; + case 78: /* db_name */ + case 79: /* table_name */ + case 80: /* column_name */ + case 81: /* function_name */ + case 82: /* table_alias */ + case 83: /* column_alias */ + case 99: /* alias_opt */ { PARSER_DESTRUCTOR_TRACE; } break; - case 73: /* join_type */ + case 89: /* compare_op */ + case 90: /* in_op */ { PARSER_DESTRUCTOR_TRACE; } break; - case 76: /* set_quantifier_opt */ + case 101: /* join_type */ { PARSER_DESTRUCTOR_TRACE; } break; - case 77: /* select_list */ - case 79: /* partition_by_clause_opt */ - case 81: /* group_by_clause_opt */ - case 83: /* select_sublist */ - case 86: /* order_by_clause_opt */ - case 90: /* sort_specification_list */ -{ - PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy40)); -} - break; - case 92: /* ordering_specification_opt */ + case 104: /* set_quantifier_opt */ { PARSER_DESTRUCTOR_TRACE; } break; - case 93: /* null_ordering_opt */ + case 115: /* fill_mode */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 123: /* ordering_specification_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 124: /* null_ordering_opt */ { PARSER_DESTRUCTOR_TRACE; } @@ -1160,112 +1289,141 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 45, -2 }, /* (0) cmd ::= SHOW DATABASES */ - { 45, -1 }, /* (1) cmd ::= query_expression */ - { 47, -1 }, /* (2) unsigned_integer ::= NK_INTEGER */ - { 48, -1 }, /* (3) unsigned_approximate_numeric ::= NK_FLOAT */ - { 49, -1 }, /* (4) signed_integer ::= unsigned_integer */ - { 49, -2 }, /* (5) signed_integer ::= NK_PLUS unsigned_integer */ - { 49, -2 }, /* (6) signed_integer ::= NK_MINUS unsigned_integer */ - { 53, -1 }, /* (7) db_name ::= NK_ID */ - { 54, -1 }, /* (8) table_name ::= NK_ID */ - { 55, -1 }, /* (9) column_name ::= NK_ID */ - { 56, -1 }, /* (10) table_alias ::= NK_ID */ - { 61, -1 }, /* (11) column_reference ::= column_name */ - { 61, -3 }, /* (12) column_reference ::= table_name NK_DOT column_name */ - { 63, -1 }, /* (13) boolean_value_expression ::= boolean_primary */ - { 63, -2 }, /* (14) boolean_value_expression ::= NOT boolean_primary */ - { 63, -3 }, /* (15) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 63, -3 }, /* (16) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 64, -1 }, /* (17) boolean_primary ::= predicate */ - { 64, -3 }, /* (18) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 66, -2 }, /* (19) from_clause ::= FROM table_reference_list */ - { 67, -1 }, /* (20) table_reference_list ::= table_reference */ - { 67, -3 }, /* (21) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 68, -1 }, /* (22) table_reference ::= table_primary */ - { 68, -1 }, /* (23) table_reference ::= joined_table */ - { 69, -2 }, /* (24) table_primary ::= table_name correlation_or_recognition_opt */ - { 69, -4 }, /* (25) table_primary ::= db_name NK_DOT table_name correlation_or_recognition_opt */ - { 69, -2 }, /* (26) table_primary ::= subquery correlation_or_recognition_opt */ - { 71, 0 }, /* (27) correlation_or_recognition_opt ::= */ - { 71, -1 }, /* (28) correlation_or_recognition_opt ::= table_alias */ - { 71, -2 }, /* (29) correlation_or_recognition_opt ::= AS table_alias */ - { 72, -3 }, /* (30) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 72, -3 }, /* (31) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 70, -6 }, /* (32) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 73, -1 }, /* (33) join_type ::= INNER */ - { 75, -9 }, /* (34) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 76, 0 }, /* (35) set_quantifier_opt ::= */ - { 76, -1 }, /* (36) set_quantifier_opt ::= DISTINCT */ - { 76, -1 }, /* (37) set_quantifier_opt ::= ALL */ - { 77, -1 }, /* (38) select_list ::= NK_STAR */ - { 77, -1 }, /* (39) select_list ::= select_sublist */ - { 83, -1 }, /* (40) select_sublist ::= select_item */ - { 83, -3 }, /* (41) select_sublist ::= select_sublist NK_COMMA select_item */ - { 84, -1 }, /* (42) select_item ::= value_expression */ - { 84, -2 }, /* (43) select_item ::= value_expression NK_ID */ - { 84, -3 }, /* (44) select_item ::= value_expression AS NK_ID */ - { 84, -3 }, /* (45) select_item ::= table_name NK_DOT NK_STAR */ - { 46, -4 }, /* (46) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 85, -1 }, /* (47) query_expression_body ::= query_primary */ - { 85, -4 }, /* (48) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 89, -1 }, /* (49) query_primary ::= query_specification */ - { 89, -6 }, /* (50) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ - { 86, 0 }, /* (51) order_by_clause_opt ::= */ - { 86, -3 }, /* (52) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 87, 0 }, /* (53) slimit_clause_opt ::= */ - { 87, -2 }, /* (54) slimit_clause_opt ::= SLIMIT signed_integer */ - { 87, -4 }, /* (55) slimit_clause_opt ::= SLIMIT signed_integer SOFFSET signed_integer */ - { 87, -4 }, /* (56) slimit_clause_opt ::= SLIMIT signed_integer NK_COMMA signed_integer */ - { 88, 0 }, /* (57) limit_clause_opt ::= */ - { 88, -2 }, /* (58) limit_clause_opt ::= LIMIT signed_integer */ - { 88, -4 }, /* (59) limit_clause_opt ::= LIMIT signed_integer OFFSET signed_integer */ - { 88, -4 }, /* (60) limit_clause_opt ::= LIMIT signed_integer NK_COMMA signed_integer */ - { 62, -3 }, /* (61) subquery ::= NK_LR query_expression NK_RP */ - { 74, -1 }, /* (62) search_condition ::= boolean_value_expression */ - { 90, -1 }, /* (63) sort_specification_list ::= sort_specification */ - { 90, -3 }, /* (64) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 91, -3 }, /* (65) sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ - { 92, 0 }, /* (66) ordering_specification_opt ::= */ - { 92, -1 }, /* (67) ordering_specification_opt ::= ASC */ - { 92, -1 }, /* (68) ordering_specification_opt ::= DESC */ - { 93, 0 }, /* (69) null_ordering_opt ::= */ - { 93, -2 }, /* (70) null_ordering_opt ::= NULLS FIRST */ - { 93, -2 }, /* (71) null_ordering_opt ::= NULLS LAST */ - { 50, -1 }, /* (72) unsigned_literal ::= unsigned_numeric_literal */ - { 50, -1 }, /* (73) unsigned_literal ::= general_literal */ - { 51, -1 }, /* (74) unsigned_numeric_literal ::= unsigned_integer */ - { 51, -1 }, /* (75) unsigned_numeric_literal ::= unsigned_approximate_numeric */ - { 52, -1 }, /* (76) general_literal ::= NK_STRING */ - { 52, -1 }, /* (77) general_literal ::= NK_BOOL */ - { 52, -1 }, /* (78) general_literal ::= NK_NOW */ - { 57, -1 }, /* (79) unsigned_value_specification ::= unsigned_literal */ - { 57, -1 }, /* (80) unsigned_value_specification ::= NK_QUESTION */ - { 58, -3 }, /* (81) value_expression_primary ::= NK_LP value_expression NK_RP */ - { 58, -1 }, /* (82) value_expression_primary ::= nonparenthesized_value_expression_primary */ - { 60, -1 }, /* (83) nonparenthesized_value_expression_primary ::= unsigned_value_specification */ - { 60, -1 }, /* (84) nonparenthesized_value_expression_primary ::= column_reference */ - { 60, -1 }, /* (85) nonparenthesized_value_expression_primary ::= subquery */ - { 69, -1 }, /* (86) table_primary ::= parenthesized_joined_table */ - { 65, 0 }, /* (87) predicate ::= */ - { 78, 0 }, /* (88) where_clause_opt ::= */ - { 79, 0 }, /* (89) partition_by_clause_opt ::= */ - { 80, 0 }, /* (90) twindow_clause_opt ::= */ - { 81, 0 }, /* (91) group_by_clause_opt ::= */ - { 82, 0 }, /* (92) having_clause_opt ::= */ - { 94, -4 }, /* (93) value_function ::= NK_ID NK_LP value_expression NK_RP */ - { 94, -6 }, /* (94) value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ - { 59, -1 }, /* (95) value_expression ::= common_value_expression */ - { 95, -1 }, /* (96) common_value_expression ::= numeric_value_expression */ - { 96, -1 }, /* (97) numeric_value_expression ::= numeric_primary */ - { 96, -2 }, /* (98) numeric_value_expression ::= NK_PLUS numeric_primary */ - { 96, -2 }, /* (99) numeric_value_expression ::= NK_MINUS numeric_primary */ - { 96, -3 }, /* (100) numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - { 96, -3 }, /* (101) numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ - { 96, -3 }, /* (102) numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ - { 96, -3 }, /* (103) numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ - { 97, -1 }, /* (104) numeric_primary ::= value_expression_primary */ - { 97, -1 }, /* (105) numeric_primary ::= value_function */ + { 73, -2 }, /* (0) cmd ::= SHOW DATABASES */ + { 73, -1 }, /* (1) cmd ::= query_expression */ + { 75, -1 }, /* (2) literal ::= NK_INTEGER */ + { 75, -1 }, /* (3) literal ::= NK_FLOAT */ + { 75, -1 }, /* (4) literal ::= NK_STRING */ + { 75, -1 }, /* (5) literal ::= NK_BOOL */ + { 75, -2 }, /* (6) literal ::= TIMESTAMP NK_STRING */ + { 75, -1 }, /* (7) literal ::= duration_literal */ + { 76, -1 }, /* (8) duration_literal ::= NK_VARIABLE */ + { 77, -1 }, /* (9) literal_list ::= literal */ + { 77, -3 }, /* (10) literal_list ::= literal_list NK_COMMA literal */ + { 78, -1 }, /* (11) db_name ::= NK_ID */ + { 79, -1 }, /* (12) table_name ::= NK_ID */ + { 80, -1 }, /* (13) column_name ::= NK_ID */ + { 81, -1 }, /* (14) function_name ::= NK_ID */ + { 82, -1 }, /* (15) table_alias ::= NK_ID */ + { 83, -1 }, /* (16) column_alias ::= NK_ID */ + { 84, -1 }, /* (17) expression ::= literal */ + { 84, -1 }, /* (18) expression ::= column_reference */ + { 84, -4 }, /* (19) expression ::= function_name NK_LP expression_list NK_RP */ + { 84, -1 }, /* (20) expression ::= subquery */ + { 84, -3 }, /* (21) expression ::= NK_LP expression NK_RP */ + { 84, -2 }, /* (22) expression ::= NK_PLUS expression */ + { 84, -2 }, /* (23) expression ::= NK_MINUS expression */ + { 84, -3 }, /* (24) expression ::= expression NK_PLUS expression */ + { 84, -3 }, /* (25) expression ::= expression NK_MINUS expression */ + { 84, -3 }, /* (26) expression ::= expression NK_STAR expression */ + { 84, -3 }, /* (27) expression ::= expression NK_SLASH expression */ + { 84, -3 }, /* (28) expression ::= expression NK_REM expression */ + { 86, -1 }, /* (29) expression_list ::= expression */ + { 86, -3 }, /* (30) expression_list ::= expression_list NK_COMMA expression */ + { 85, -1 }, /* (31) column_reference ::= column_name */ + { 85, -3 }, /* (32) column_reference ::= table_name NK_DOT column_name */ + { 88, -3 }, /* (33) predicate ::= expression compare_op expression */ + { 88, -5 }, /* (34) predicate ::= expression BETWEEN expression AND expression */ + { 88, -6 }, /* (35) predicate ::= expression NOT BETWEEN expression AND expression */ + { 88, -3 }, /* (36) predicate ::= expression IS NULL */ + { 88, -4 }, /* (37) predicate ::= expression IS NOT NULL */ + { 88, -3 }, /* (38) predicate ::= expression in_op in_predicate_value */ + { 89, -1 }, /* (39) compare_op ::= NK_LT */ + { 89, -1 }, /* (40) compare_op ::= NK_GT */ + { 89, -1 }, /* (41) compare_op ::= NK_LE */ + { 89, -1 }, /* (42) compare_op ::= NK_GE */ + { 89, -1 }, /* (43) compare_op ::= NK_NE */ + { 89, -1 }, /* (44) compare_op ::= NK_EQ */ + { 89, -1 }, /* (45) compare_op ::= LIKE */ + { 89, -2 }, /* (46) compare_op ::= NOT LIKE */ + { 89, -1 }, /* (47) compare_op ::= MATCH */ + { 89, -1 }, /* (48) compare_op ::= NMATCH */ + { 90, -1 }, /* (49) in_op ::= IN */ + { 90, -2 }, /* (50) in_op ::= NOT IN */ + { 91, -3 }, /* (51) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 92, -1 }, /* (52) boolean_value_expression ::= boolean_primary */ + { 92, -2 }, /* (53) boolean_value_expression ::= NOT boolean_primary */ + { 92, -3 }, /* (54) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 92, -3 }, /* (55) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 93, -1 }, /* (56) boolean_primary ::= predicate */ + { 93, -3 }, /* (57) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 94, -2 }, /* (58) from_clause ::= FROM table_reference_list */ + { 95, -1 }, /* (59) table_reference_list ::= table_reference */ + { 95, -3 }, /* (60) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 96, -1 }, /* (61) table_reference ::= table_primary */ + { 96, -1 }, /* (62) table_reference ::= joined_table */ + { 97, -2 }, /* (63) table_primary ::= table_name alias_opt */ + { 97, -4 }, /* (64) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 97, -2 }, /* (65) table_primary ::= subquery alias_opt */ + { 99, 0 }, /* (66) alias_opt ::= */ + { 99, -1 }, /* (67) alias_opt ::= table_alias */ + { 99, -2 }, /* (68) alias_opt ::= AS table_alias */ + { 100, -3 }, /* (69) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 100, -3 }, /* (70) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 98, -6 }, /* (71) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 101, -1 }, /* (72) join_type ::= INNER */ + { 103, -9 }, /* (73) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 104, 0 }, /* (74) set_quantifier_opt ::= */ + { 104, -1 }, /* (75) set_quantifier_opt ::= DISTINCT */ + { 104, -1 }, /* (76) set_quantifier_opt ::= ALL */ + { 105, -1 }, /* (77) select_list ::= NK_STAR */ + { 105, -1 }, /* (78) select_list ::= select_sublist */ + { 111, -1 }, /* (79) select_sublist ::= select_item */ + { 111, -3 }, /* (80) select_sublist ::= select_sublist NK_COMMA select_item */ + { 112, -1 }, /* (81) select_item ::= expression */ + { 112, -2 }, /* (82) select_item ::= expression column_alias */ + { 112, -3 }, /* (83) select_item ::= expression AS column_alias */ + { 112, -3 }, /* (84) select_item ::= table_name NK_DOT NK_STAR */ + { 106, 0 }, /* (85) where_clause_opt ::= */ + { 106, -2 }, /* (86) where_clause_opt ::= WHERE search_condition */ + { 107, 0 }, /* (87) partition_by_clause_opt ::= */ + { 107, -3 }, /* (88) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 108, 0 }, /* (89) twindow_clause_opt ::= */ + { 108, -6 }, /* (90) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 108, -4 }, /* (91) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 108, -6 }, /* (92) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 108, -8 }, /* (93) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 113, 0 }, /* (94) sliding_opt ::= */ + { 113, -4 }, /* (95) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 114, 0 }, /* (96) fill_opt ::= */ + { 114, -4 }, /* (97) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 114, -6 }, /* (98) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 115, -1 }, /* (99) fill_mode ::= NONE */ + { 115, -1 }, /* (100) fill_mode ::= PREV */ + { 115, -1 }, /* (101) fill_mode ::= NULL */ + { 115, -1 }, /* (102) fill_mode ::= LINEAR */ + { 115, -1 }, /* (103) fill_mode ::= NEXT */ + { 109, 0 }, /* (104) group_by_clause_opt ::= */ + { 109, -3 }, /* (105) group_by_clause_opt ::= GROUP BY expression_list */ + { 110, 0 }, /* (106) having_clause_opt ::= */ + { 110, -2 }, /* (107) having_clause_opt ::= HAVING search_condition */ + { 74, -4 }, /* (108) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 116, -1 }, /* (109) query_expression_body ::= query_primary */ + { 116, -4 }, /* (110) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 120, -1 }, /* (111) query_primary ::= query_specification */ + { 120, -6 }, /* (112) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ + { 117, 0 }, /* (113) order_by_clause_opt ::= */ + { 117, -3 }, /* (114) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 118, 0 }, /* (115) slimit_clause_opt ::= */ + { 118, -2 }, /* (116) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 118, -4 }, /* (117) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 118, -4 }, /* (118) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 119, 0 }, /* (119) limit_clause_opt ::= */ + { 119, -2 }, /* (120) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 119, -4 }, /* (121) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 119, -4 }, /* (122) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 87, -3 }, /* (123) subquery ::= NK_LR query_expression NK_RP */ + { 102, -1 }, /* (124) search_condition ::= boolean_value_expression */ + { 121, -1 }, /* (125) sort_specification_list ::= sort_specification */ + { 121, -3 }, /* (126) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 122, -3 }, /* (127) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 123, 0 }, /* (128) ordering_specification_opt ::= */ + { 123, -1 }, /* (129) ordering_specification_opt ::= ASC */ + { 123, -1 }, /* (130) ordering_specification_opt ::= DESC */ + { 124, 0 }, /* (131) null_ordering_opt ::= */ + { 124, -2 }, /* (132) null_ordering_opt ::= NULLS FIRST */ + { 124, -2 }, /* (133) null_ordering_opt ::= NULLS LAST */ + { 97, -1 }, /* (134) table_primary ::= parenthesized_joined_table */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1356,347 +1514,376 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy168; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy212; } break; - case 2: /* unsigned_integer ::= NK_INTEGER */ -{ yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 2: /* literal ::= NK_INTEGER */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 3: /* unsigned_approximate_numeric ::= NK_FLOAT */ -{ yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 3: /* literal ::= NK_FLOAT */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 4: /* signed_integer ::= unsigned_integer */ -{ yylhsminor.yy168 = yymsp[0].minor.yy168; } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 4: /* literal ::= NK_STRING */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 5: /* signed_integer ::= NK_PLUS unsigned_integer */ -{ yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } + case 5: /* literal ::= NK_BOOL */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 6: /* signed_integer ::= NK_MINUS unsigned_integer */ -{ yymsp[-1].minor.yy168 = addMinusSign(pCxt, yymsp[0].minor.yy168);} + case 6: /* literal ::= TIMESTAMP NK_STRING */ +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 7: /* db_name ::= NK_ID */ - case 8: /* table_name ::= NK_ID */ yytestcase(yyruleno==8); - case 9: /* column_name ::= NK_ID */ yytestcase(yyruleno==9); - case 10: /* table_alias ::= NK_ID */ yytestcase(yyruleno==10); -{ PARSER_TRACE; yylhsminor.yy169 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy169 = yylhsminor.yy169; + case 7: /* literal ::= duration_literal */ + case 17: /* expression ::= literal */ yytestcase(yyruleno==17); + case 18: /* expression ::= column_reference */ yytestcase(yyruleno==18); + case 20: /* expression ::= subquery */ yytestcase(yyruleno==20); + case 52: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==52); + case 56: /* boolean_primary ::= predicate */ yytestcase(yyruleno==56); + case 59: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==59); + case 61: /* table_reference ::= table_primary */ yytestcase(yyruleno==61); + case 62: /* table_reference ::= joined_table */ yytestcase(yyruleno==62); + case 81: /* select_item ::= expression */ yytestcase(yyruleno==81); + case 109: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==109); + case 111: /* query_primary ::= query_specification */ yytestcase(yyruleno==111); + case 124: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==124); +{ PARSER_TRACE; yylhsminor.yy212 = yymsp[0].minor.yy212; } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 11: /* column_reference ::= column_name */ -{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy169); } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 8: /* duration_literal ::= NK_VARIABLE */ +{ PARSER_TRACE; yylhsminor.yy212 = createDurationValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 12: /* column_reference ::= table_name NK_DOT column_name */ -{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy169, &yymsp[0].minor.yy169); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 9: /* literal_list ::= literal */ + case 29: /* expression_list ::= expression */ yytestcase(yyruleno==29); + case 79: /* select_sublist ::= select_item */ yytestcase(yyruleno==79); + case 125: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy212); } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 13: /* boolean_value_expression ::= boolean_primary */ - case 17: /* boolean_primary ::= predicate */ yytestcase(yyruleno==17); - case 20: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==20); - case 22: /* table_reference ::= table_primary */ yytestcase(yyruleno==22); - case 23: /* table_reference ::= joined_table */ yytestcase(yyruleno==23); - case 42: /* select_item ::= value_expression */ yytestcase(yyruleno==42); - case 47: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==47); - case 49: /* query_primary ::= query_specification */ yytestcase(yyruleno==49); - case 62: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==62); -{ PARSER_TRACE; yylhsminor.yy168 = yymsp[0].minor.yy168; } - yymsp[0].minor.yy168 = yylhsminor.yy168; + case 10: /* literal_list ::= literal_list NK_COMMA literal */ + case 30: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==30); + case 80: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==80); + case 126: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; break; - case 14: /* boolean_value_expression ::= NOT boolean_primary */ -{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy168, NULL); } + case 11: /* db_name ::= NK_ID */ + case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); + case 13: /* column_name ::= NK_ID */ yytestcase(yyruleno==13); + case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); + case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); + case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy79 = yylhsminor.yy79; break; - case 15: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yylhsminor.yy212 = createFunctionNode(pCxt, &yymsp[-3].minor.yy79, yymsp[-1].minor.yy174); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 16: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 21: /* expression ::= NK_LP expression NK_RP */ + case 57: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==57); + case 69: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==69); + case 70: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==70); + case 123: /* subquery ::= NK_LR query_expression NK_RP */ yytestcase(yyruleno==123); +{ PARSER_TRACE; yymsp[-2].minor.yy212 = yymsp[-1].minor.yy212; } break; - case 18: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - case 30: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==30); - case 31: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==31); - case 61: /* subquery ::= NK_LR query_expression NK_RP */ yytestcase(yyruleno==61); -{ PARSER_TRACE; yymsp[-2].minor.yy168 = yymsp[-1].minor.yy168; } + case 22: /* expression ::= NK_PLUS expression */ + case 58: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==58); + case 86: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==86); + case 107: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==107); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = yymsp[0].minor.yy212; } break; - case 19: /* from_clause ::= FROM table_reference_list */ -{ PARSER_TRACE; yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } + case 23: /* expression ::= NK_MINUS expression */ +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createOperatorNode(pCxt, OP_TYPE_SUB, yymsp[0].minor.yy212, NULL); } break; - case 21: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy168, yymsp[0].minor.yy168, NULL); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 24: /* expression ::= expression NK_PLUS expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_ADD, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 24: /* table_primary ::= table_name correlation_or_recognition_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy169, &yymsp[0].minor.yy169); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + case 25: /* expression ::= expression NK_MINUS expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_SUB, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 25: /* table_primary ::= db_name NK_DOT table_name correlation_or_recognition_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, &yymsp[-3].minor.yy169, &yymsp[-1].minor.yy169, &yymsp[0].minor.yy169); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + case 26: /* expression ::= expression NK_STAR expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_MULTI, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 26: /* table_primary ::= subquery correlation_or_recognition_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createTempTableNode(pCxt, yymsp[-1].minor.yy168, &yymsp[0].minor.yy169); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + case 27: /* expression ::= expression NK_SLASH expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_DIV, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 27: /* correlation_or_recognition_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy169 = nil_token; } + case 28: /* expression ::= expression NK_REM expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_MOD, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 28: /* correlation_or_recognition_opt ::= table_alias */ -{ PARSER_TRACE; yylhsminor.yy169 = yymsp[0].minor.yy169; } - yymsp[0].minor.yy169 = yylhsminor.yy169; + case 31: /* column_reference ::= column_name */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy79); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 29: /* correlation_or_recognition_opt ::= AS table_alias */ -{ PARSER_TRACE; yymsp[-1].minor.yy169 = yymsp[0].minor.yy169; } + case 32: /* column_reference ::= table_name NK_DOT column_name */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 32: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, yymsp[-4].minor.yy148, yymsp[-5].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-5].minor.yy168 = yylhsminor.yy168; + case 33: /* predicate ::= expression compare_op expression */ + case 38: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==38); +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 33: /* join_type ::= INNER */ -{ PARSER_TRACE; yymsp[0].minor.yy148 = JOIN_TYPE_INNER; } + case 34: /* predicate ::= expression BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createBetweenAnd(pCxt, yymsp[-4].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-4].minor.yy212 = yylhsminor.yy212; break; - case 34: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 35: /* predicate ::= expression NOT BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createNotBetweenAnd(pCxt, yymsp[-2].minor.yy212, yymsp[-5].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; + break; + case 36: /* predicate ::= expression IS NULL */ +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, yymsp[-2].minor.yy212, true); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 37: /* predicate ::= expression IS NOT NULL */ +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, yymsp[-3].minor.yy212, false); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; + break; + case 39: /* compare_op ::= NK_LT */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_THAN; } + break; + case 40: /* compare_op ::= NK_GT */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_THAN; } + break; + case 41: /* compare_op ::= NK_LE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_EQUAL; } + break; + case 42: /* compare_op ::= NK_GE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_EQUAL; } + break; + case 43: /* compare_op ::= NK_NE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NOT_EQUAL; } + break; + case 44: /* compare_op ::= NK_EQ */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_EQUAL; } + break; + case 45: /* compare_op ::= LIKE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LIKE; } + break; + case 46: /* compare_op ::= NOT LIKE */ +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_LIKE; } + break; + case 47: /* compare_op ::= MATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_MATCH; } + break; + case 48: /* compare_op ::= NMATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NMATCH; } + break; + case 49: /* in_op ::= IN */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_IN; } + break; + case 50: /* in_op ::= NOT IN */ +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_IN; } + break; + case 51: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yymsp[-2].minor.yy212 = createNodeListNode(pCxt, yymsp[-1].minor.yy174); } + break; + case 53: /* boolean_value_expression ::= NOT boolean_primary */ +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy212, NULL); } + break; + case 54: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 55: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 60: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy212, yymsp[0].minor.yy212, NULL); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 63: /* table_primary ::= table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; + break; + case 64: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, &yymsp[-3].minor.yy79, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; + break; + case 65: /* table_primary ::= subquery alias_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createTempTableNode(pCxt, yymsp[-1].minor.yy212, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; + break; + case 66: /* alias_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy79 = nil_token; } + break; + case 67: /* alias_opt ::= table_alias */ +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy79; } + yymsp[0].minor.yy79 = yylhsminor.yy79; + break; + case 68: /* alias_opt ::= AS table_alias */ +{ PARSER_TRACE; yymsp[-1].minor.yy79 = yymsp[0].minor.yy79; } + break; + case 71: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, yymsp[-4].minor.yy162, yymsp[-5].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; + break; + case 72: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy162 = JOIN_TYPE_INNER; } + break; + case 73: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { PARSER_TRACE; - yymsp[-8].minor.yy168 = createSelectStmt(pCxt, yymsp[-7].minor.yy173, yymsp[-6].minor.yy40, yymsp[-5].minor.yy168); - yymsp[-8].minor.yy168 = addWhereClause(pCxt, yymsp[-8].minor.yy168, yymsp[-4].minor.yy168); - yymsp[-8].minor.yy168 = addPartitionByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-3].minor.yy40); - yymsp[-8].minor.yy168 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy168, yymsp[-2].minor.yy168); - yymsp[-8].minor.yy168 = addGroupByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-1].minor.yy40); - yymsp[-8].minor.yy168 = addHavingClause(pCxt, yymsp[-8].minor.yy168, yymsp[0].minor.yy168); + yymsp[-8].minor.yy212 = createSelectStmt(pCxt, yymsp[-7].minor.yy237, yymsp[-6].minor.yy174, yymsp[-5].minor.yy212); + yymsp[-8].minor.yy212 = addWhereClause(pCxt, yymsp[-8].minor.yy212, yymsp[-4].minor.yy212); + yymsp[-8].minor.yy212 = addPartitionByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-3].minor.yy174); + yymsp[-8].minor.yy212 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy212, yymsp[-2].minor.yy212); + yymsp[-8].minor.yy212 = addGroupByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-1].minor.yy174); + yymsp[-8].minor.yy212 = addHavingClause(pCxt, yymsp[-8].minor.yy212, yymsp[0].minor.yy212); } break; - case 35: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy173 = false; } + case 74: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy237 = false; } break; - case 36: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy173 = true; } + case 75: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = true; } break; - case 37: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy173 = false; } + case 76: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = false; } break; - case 38: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = NULL; } + case 77: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy174 = NULL; } break; - case 39: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy40 = yymsp[0].minor.yy40; } - yymsp[0].minor.yy40 = yylhsminor.yy40; + case 78: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy174 = yymsp[0].minor.yy174; } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 40: /* select_sublist ::= select_item */ - case 63: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==63); -{ PARSER_TRACE; yylhsminor.yy40 = createNodeList(pCxt, yymsp[0].minor.yy168); } - yymsp[0].minor.yy40 = yylhsminor.yy40; + case 82: /* select_item ::= expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, yymsp[-1].minor.yy212, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; - case 41: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 64: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==64); -{ PARSER_TRACE; yylhsminor.yy40 = addNodeToList(pCxt, yymsp[-2].minor.yy40, yymsp[0].minor.yy168); } - yymsp[-2].minor.yy40 = yylhsminor.yy40; + case 83: /* select_item ::= expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, yymsp[-2].minor.yy212, &yymsp[0].minor.yy79); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 43: /* select_item ::= value_expression NK_ID */ -{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, yymsp[-1].minor.yy168, &yymsp[0].minor.yy0); } - yymsp[-1].minor.yy168 = yylhsminor.yy168; + case 84: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 44: /* select_item ::= value_expression AS NK_ID */ -{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, yymsp[-2].minor.yy168, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 85: /* where_clause_opt ::= */ + case 89: /* twindow_clause_opt ::= */ yytestcase(yyruleno==89); + case 94: /* sliding_opt ::= */ yytestcase(yyruleno==94); + case 96: /* fill_opt ::= */ yytestcase(yyruleno==96); + case 106: /* having_clause_opt ::= */ yytestcase(yyruleno==106); + case 115: /* slimit_clause_opt ::= */ yytestcase(yyruleno==115); + case 119: /* limit_clause_opt ::= */ yytestcase(yyruleno==119); +{ PARSER_TRACE; yymsp[1].minor.yy212 = NULL; } break; - case 45: /* select_item ::= table_name NK_DOT NK_STAR */ -{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy169, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 87: /* partition_by_clause_opt ::= */ + case 104: /* group_by_clause_opt ::= */ yytestcase(yyruleno==104); + case 113: /* order_by_clause_opt ::= */ yytestcase(yyruleno==113); +{ PARSER_TRACE; yymsp[1].minor.yy174 = NULL; } break; - case 46: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 88: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 105: /* group_by_clause_opt ::= GROUP BY expression_list */ yytestcase(yyruleno==105); + case 114: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==114); +{ PARSER_TRACE; yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } + break; + case 90: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createSessionWindowNode(pCxt, yymsp[-3].minor.yy212, &yymsp[-1].minor.yy0); } + break; + case 91: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createStateWindowNode(pCxt, yymsp[-1].minor.yy212); } + break; + case 92: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy212, NULL, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } + break; + case 93: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy212, yymsp[-3].minor.yy212, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } + break; + case 95: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = yymsp[-1].minor.yy212; } + break; + case 97: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createFillNode(pCxt, yymsp[-1].minor.yy44, NULL); } + break; + case 98: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } + break; + case 99: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NONE; } + break; + case 100: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_PREV; } + break; + case 101: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NULL; } + break; + case 102: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_LINEAR; } + break; + case 103: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NEXT; } + break; + case 108: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { PARSER_TRACE; - yylhsminor.yy168 = addOrderByClause(pCxt, yymsp[-3].minor.yy168, yymsp[-2].minor.yy40); - yylhsminor.yy168 = addSlimitClause(pCxt, yylhsminor.yy168, yymsp[-1].minor.yy168); - yylhsminor.yy168 = addLimitClause(pCxt, yylhsminor.yy168, yymsp[0].minor.yy168); + yylhsminor.yy212 = addOrderByClause(pCxt, yymsp[-3].minor.yy212, yymsp[-2].minor.yy174); + yylhsminor.yy212 = addSlimitClause(pCxt, yylhsminor.yy212, yymsp[-1].minor.yy212); + yylhsminor.yy212 = addLimitClause(pCxt, yylhsminor.yy212, yymsp[0].minor.yy212); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 48: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy168 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy168, yymsp[0].minor.yy168); } - yymsp[-3].minor.yy168 = yylhsminor.yy168; + case 110: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy212 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 50: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy168 = yymsp[-4].minor.yy168;} - yy_destructor(yypParser,86,&yymsp[-3].minor); - yy_destructor(yypParser,88,&yymsp[-2].minor); - yy_destructor(yypParser,87,&yymsp[-1].minor); + case 112: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = yymsp[-4].minor.yy212;} + yy_destructor(yypParser,117,&yymsp[-3].minor); + yy_destructor(yypParser,119,&yymsp[-2].minor); + yy_destructor(yypParser,118,&yymsp[-1].minor); break; - case 51: /* order_by_clause_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy40 = NULL; } + case 116: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 120: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==120); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 52: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ -{ PARSER_TRACE; yymsp[-2].minor.yy40 = yymsp[0].minor.yy40; } + case 117: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 121: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==121); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 53: /* slimit_clause_opt ::= */ - case 57: /* limit_clause_opt ::= */ yytestcase(yyruleno==57); -{ PARSER_TRACE; yymsp[1].minor.yy168 = NULL; } + case 118: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 122: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==122); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 54: /* slimit_clause_opt ::= SLIMIT signed_integer */ - case 58: /* limit_clause_opt ::= LIMIT signed_integer */ yytestcase(yyruleno==58); -{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLimitNode(pCxt, yymsp[0].minor.yy168, 0); } + case 127: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy212, yymsp[-1].minor.yy188, yymsp[0].minor.yy107); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 55: /* slimit_clause_opt ::= SLIMIT signed_integer SOFFSET signed_integer */ - case 59: /* limit_clause_opt ::= LIMIT signed_integer OFFSET signed_integer */ yytestcase(yyruleno==59); -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + case 128: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy188 = ORDER_ASC; } break; - case 56: /* slimit_clause_opt ::= SLIMIT signed_integer NK_COMMA signed_integer */ - case 60: /* limit_clause_opt ::= LIMIT signed_integer NK_COMMA signed_integer */ yytestcase(yyruleno==60); -{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, yymsp[0].minor.yy168, yymsp[-2].minor.yy168); } + case 129: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_ASC; } break; - case 65: /* sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy168 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy168, yymsp[-1].minor.yy106, yymsp[0].minor.yy193); } - yymsp[-2].minor.yy168 = yylhsminor.yy168; + case 130: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_DESC; } break; - case 66: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy106 = ORDER_ASC; } + case 131: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy107 = NULL_ORDER_DEFAULT; } break; - case 67: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy106 = ORDER_ASC; } + case 132: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_FIRST; } break; - case 68: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy106 = ORDER_DESC; } + case 133: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_LAST; } break; - case 69: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy193 = NULL_ORDER_DEFAULT; } - break; - case 70: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy193 = NULL_ORDER_FIRST; } - break; - case 71: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy193 = NULL_ORDER_LAST; } - break; - case 72: /* unsigned_literal ::= unsigned_numeric_literal */ -{ yy_destructor(yypParser,51,&yymsp[0].minor); -{ -} -} - break; - case 73: /* unsigned_literal ::= general_literal */ -{ yy_destructor(yypParser,52,&yymsp[0].minor); -{ -} -} - break; - case 74: /* unsigned_numeric_literal ::= unsigned_integer */ -{ yy_destructor(yypParser,47,&yymsp[0].minor); -{ -} -} - break; - case 75: /* unsigned_numeric_literal ::= unsigned_approximate_numeric */ -{ yy_destructor(yypParser,48,&yymsp[0].minor); -{ -} -} - break; - case 79: /* unsigned_value_specification ::= unsigned_literal */ -{ yy_destructor(yypParser,50,&yymsp[0].minor); -{ -} -} - break; - case 81: /* value_expression_primary ::= NK_LP value_expression NK_RP */ - case 93: /* value_function ::= NK_ID NK_LP value_expression NK_RP */ yytestcase(yyruleno==93); -{ -} - yy_destructor(yypParser,59,&yymsp[-1].minor); - break; - case 82: /* value_expression_primary ::= nonparenthesized_value_expression_primary */ -{ yy_destructor(yypParser,60,&yymsp[0].minor); -{ -} -} - break; - case 83: /* nonparenthesized_value_expression_primary ::= unsigned_value_specification */ -{ yy_destructor(yypParser,57,&yymsp[0].minor); -{ -} -} - break; - case 84: /* nonparenthesized_value_expression_primary ::= column_reference */ -{ yy_destructor(yypParser,61,&yymsp[0].minor); -{ -} -} - break; - case 85: /* nonparenthesized_value_expression_primary ::= subquery */ -{ yy_destructor(yypParser,62,&yymsp[0].minor); -{ -} -} - break; - case 86: /* table_primary ::= parenthesized_joined_table */ -{ yy_destructor(yypParser,72,&yymsp[0].minor); -{ -} -} - break; - case 94: /* value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ -{ -} - yy_destructor(yypParser,59,&yymsp[-3].minor); - yy_destructor(yypParser,59,&yymsp[-1].minor); - break; - case 95: /* value_expression ::= common_value_expression */ -{ yy_destructor(yypParser,95,&yymsp[0].minor); -{ -} -} - break; - case 96: /* common_value_expression ::= numeric_value_expression */ -{ yy_destructor(yypParser,96,&yymsp[0].minor); -{ -} -} - break; - case 97: /* numeric_value_expression ::= numeric_primary */ -{ yy_destructor(yypParser,97,&yymsp[0].minor); -{ -} -} - break; - case 98: /* numeric_value_expression ::= NK_PLUS numeric_primary */ - case 99: /* numeric_value_expression ::= NK_MINUS numeric_primary */ yytestcase(yyruleno==99); -{ -} - yy_destructor(yypParser,97,&yymsp[0].minor); - break; - case 100: /* numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - case 101: /* numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ yytestcase(yyruleno==101); - case 102: /* numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ yytestcase(yyruleno==102); - case 103: /* numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ yytestcase(yyruleno==103); -{ yy_destructor(yypParser,96,&yymsp[-2].minor); -{ -} - yy_destructor(yypParser,96,&yymsp[0].minor); -} - break; - case 104: /* numeric_primary ::= value_expression_primary */ -{ yy_destructor(yypParser,58,&yymsp[0].minor); -{ -} -} - break; - case 105: /* numeric_primary ::= value_function */ -{ yy_destructor(yypParser,94,&yymsp[0].minor); + case 134: /* table_primary ::= parenthesized_joined_table */ +{ yy_destructor(yypParser,100,&yymsp[0].minor); { } } break; default: - /* (76) general_literal ::= NK_STRING */ yytestcase(yyruleno==76); - /* (77) general_literal ::= NK_BOOL */ yytestcase(yyruleno==77); - /* (78) general_literal ::= NK_NOW */ yytestcase(yyruleno==78); - /* (80) unsigned_value_specification ::= NK_QUESTION */ yytestcase(yyruleno==80); - /* (87) predicate ::= */ yytestcase(yyruleno==87); - /* (88) where_clause_opt ::= */ yytestcase(yyruleno==88); - /* (89) partition_by_clause_opt ::= */ yytestcase(yyruleno==89); - /* (90) twindow_clause_opt ::= */ yytestcase(yyruleno==90); - /* (91) group_by_clause_opt ::= */ yytestcase(yyruleno==91); - /* (92) having_clause_opt ::= */ yytestcase(yyruleno==92); break; /********** End reduce actions ************************************************/ }; @@ -1797,7 +1984,7 @@ static void yy_accept( /* Here code is inserted which will be executed whenever the ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ - printf("parsing complete!\n" ); + PARSER_COMPLETE; /*********** End %parse_accept code *******************************************/ NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ NewParseCTX_STORE diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 09675d0917..fdedafcbb4 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -86,15 +86,15 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { int32_t i = 0; while (1) { SToken t0 = {0}; - printf("===========================\n"); + // printf("===========================\n"); if (cxt.pQueryCxt->pSql[i] == 0) { NewParse(pParser, 0, t0, &cxt); goto abort_parse; } - printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); + // printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); t0.z = (char *)(cxt.pQueryCxt->pSql + i); - printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); + // printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); i += t0.n; switch (t0.type) { @@ -131,7 +131,7 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { } abort_parse: - printf("doParse completed.\n"); + // printf("doParse completed.\n"); NewParseFree(pParser, free); destroyAstCreateContext(&cxt); pQuery->pRoot = cxt.pRootNode; diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 395b61ea20..706deb443c 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -41,19 +41,10 @@ protected: bool run(int32_t expectCode = TSDB_CODE_SUCCESS) { int32_t code = doParse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { - cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; return (code == expectCode); } - cout << nodeType(query_.pRoot) << endl; if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { - // SNode* pWhereCond; - // SNodeList* pPartitionByList; // SNode - // SNode* pWindowClause; - // SNodeList* pGroupByList; // SGroupingSetNode - // SNodeList* pOrderByList; // SOrderByExprNode - // SLimitNode limit; - // SLimitNode slimit; - SSelectStmt* select = (SSelectStmt*)query_.pRoot; string sql("SELECT "); if (select->isDistinct) { @@ -68,14 +59,6 @@ protected: tableToSql(select->pFromTable, sql); cout << sql << endl; } - // char* pStr = NULL; - // int32_t len = 0; - // code = nodesNodeToString(query_.pRoot, &pStr, &len); - // if (code != TSDB_CODE_SUCCESS) { - // cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - // return code; - // } - // cout << "node tree:\n" << pStr << endl; return (code == expectCode); } @@ -85,7 +68,6 @@ private: void tableToSql(const SNode* node, string& sql) { const STableNode* table = (const STableNode*)node; - cout << "node : " << nodeType(node) << endl; switch (nodeType(node)) { case QUERY_NODE_REAL_TABLE: { SRealTableNode* realTable = (SRealTableNode*)table; @@ -108,12 +90,14 @@ private: if (!firstNode) { sql.append(", "); } + firstNode = false; switch (nodeType(node)) { case QUERY_NODE_COLUMN: sql.append(((SColumnNode*)node)->colName); break; } } + sql.append(" "); } void reset() { @@ -140,6 +124,12 @@ TEST_F(NewParserTest, selectStar) { bind("SELECT * FROM test.t1"); ASSERT_TRUE(run()); + + bind("SELECT ts FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT ts, tag1, c1 FROM t1"); + ASSERT_TRUE(run()); } TEST_F(NewParserTest, syntaxError) { diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/nodes/src/nodesEqualFuncs.c index f2752fe7f9..41d1e5b05d 100644 --- a/source/nodes/src/nodesEqualFuncs.c +++ b/source/nodes/src/nodesEqualFuncs.c @@ -91,7 +91,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCondNode* b) { COMPARE_NODE_FIELD(pExpr); - COMPARE_SCALAR_FIELD(isNot); + COMPARE_SCALAR_FIELD(isNull); return true; } diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index a0384d1674..bf4c4a83cc 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -92,7 +92,12 @@ SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { return pList; } p->pNode = pNode; - pList->pTail->pNext = p; + if (NULL == pList->pHead) { + pList->pHead = p; + } + if (NULL != pList->pTail) { + pList->pTail->pNext = p; + } pList->pTail = p; return pList; } From ba13d6daff679f55140d585001b579bc49fff246 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Jan 2022 09:29:17 +0800 Subject: [PATCH 16/42] drop stb --- source/dnode/mgmt/impl/test/vnode/vnode.cpp | 18 ++-- source/dnode/mnode/impl/src/mndStb.c | 97 ++++++++++++++------- source/dnode/vnode/src/vnd/vnodeWrite.c | 5 ++ 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 17092ed737..8e56c31ed1 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -235,20 +235,26 @@ TEST_F(DndTestVnode, 04_ALTER_Stb) { } TEST_F(DndTestVnode, 05_DROP_Stb) { -#if 0 { + int32_t contLen = sizeof(SVDropTbReq); + SVDropTbReq* pReq = (SVDropTbReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "stb1"); + pReq->suid = 0; + + SMsgHead* pMsgHead = (SMsgHead*)&pReq->head; + pMsgHead->contLen = htonl(contLen); + pMsgHead->vgId = htonl(2); + for (int i = 0; i < 3; ++i) { SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, pReq, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); - } + } // else { + // ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); + //} } } -#endif } TEST_F(DndTestVnode, 06_DROP_Vnode) { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 45b8428663..450a318b26 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -181,23 +181,26 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); - atomic_exchange_32(&pOld->updateTime, pNew->updateTime); - atomic_exchange_32(&pOld->version, pNew->version); taosWLockLatch(&pOld->lock); - pOld->numOfColumns = pNew->numOfColumns; - pOld->numOfTags = pNew->numOfTags; int32_t totalCols = pNew->numOfTags + pNew->numOfColumns; int32_t totalSize = totalCols * sizeof(SSchema); - if (pOld->numOfTags + pOld->numOfColumns < totalCols) { void *pSchema = malloc(totalSize); if (pSchema != NULL) { free(pOld->pSchema); pOld->pSchema = pSchema; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); } } + pOld->updateTime = pNew->updateTime; + pOld->version = pNew->version; + pOld->numOfColumns = pNew->numOfColumns; + pOld->numOfTags = pNew->numOfTags; memcpy(pOld->pSchema, pNew->pSchema, totalSize); taosWUnLockLatch(&pOld->lock); return 0; @@ -228,10 +231,10 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { } static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SVCreateTbReq req; - void *buf; - int32_t bsize; - SMsgHead *pMsgHead; + SVCreateTbReq req = {0}; + void *buf = NULL; + int32_t bsize = 0; + SMsgHead *pMsgHead = NULL; req.ver = 0; SName name = {0}; @@ -358,7 +361,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - int32_t contLen; + int32_t contLen; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); @@ -498,9 +501,9 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { } // topic should have different name with stb - SStbObj *pTopic = mndAcquireStb(pMnode, pCreate->name); - if (pTopic != NULL) { - sdbRelease(pMnode->pSdb, pTopic); + SStbObj *pTopicStb = mndAcquireStb(pMnode, pCreate->name); + if (pTopicStb != NULL) { + mndReleaseStb(pMnode, pTopicStb); terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -517,7 +520,6 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { mndReleaseDb(pMnode, pDb); if (code != 0) { - terrno = code; mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -603,15 +605,6 @@ static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pS return 0; } -static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); if (pCommitRaw == NULL) return -1; @@ -621,22 +614,53 @@ static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj * return 0; } -static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } +static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; -static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_STB; + action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); - if (pTrans == NULL)goto DROP_STB_OVER; + if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; + if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto DROP_STB_OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_STB_OVER; code = 0; @@ -664,7 +688,16 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { } } - int32_t code = mndDropStb(pMnode, pReq, pStb); + SDbObj *pDb = mndAcquireDbByStb(pMnode, pDrop->name); + if (pDb == NULL) { + mndReleaseStb(pMnode, pStb); + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + int32_t code = mndDropStb(pMnode, pReq, pDb, pStb); + mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); if (code != 0) { @@ -856,7 +889,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pStb->dbUid != pDb->uid) { if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%"PRIu64, pStb->name, pDb->name, pDb->uid); + mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); } sdbRelease(pSdb, pStb); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 326d99ddbb..1f5513f581 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -103,7 +103,12 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayDestroy(vCreateTbBatchReq.pArray); break; + case TDMT_VND_ALTER_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { // // TODO: handle error From fd44bdbf579f9c53ba47f37ae9dfe5ea2485d848 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 10:36:31 +0800 Subject: [PATCH 17/42] implement get vg info --- include/client/taos.h | 15 +++-- include/util/taoserror.h | 1 + source/client/src/clientImpl.c | 19 +++--- source/client/test/clientTests.cpp | 60 +++++++++++++++-- source/dnode/mnode/impl/src/mndSubscribe.c | 77 +++++++++++++--------- source/dnode/vnode/src/tq/tq.c | 10 ++- source/libs/executor/src/executorimpl.c | 4 +- source/util/src/terror.c | 3 + 8 files changed, 128 insertions(+), 61 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 626264d94e..5f7cd12963 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -204,12 +204,19 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); +/* --------------------------TMQ INTERFACE------------------------------- */ +typedef struct tmq_resp_err_t tmq_resp_err_t; +typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; +typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; + +typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); DLL_EXPORT tmq_list_t* tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*); DLL_EXPORT tmq_conf_t* tmq_conf_new(); DLL_EXPORT int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb); DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); @@ -217,13 +224,7 @@ DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); -DLL_EXPORT tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time); - -DLL_EXPORT int32_t tmq_topic_num(tmq_message_t* msg); -DLL_EXPORT char* tmq_get_topic(tmq_message_topic_t* msg); -DLL_EXPORT int32_t tmq_get_vgId(tmq_message_topic_t* msg); -DLL_EXPORT tmq_message_tb_t* tmq_get_next_tb(tmq_message_topic_t* msg, tmq_tb_iter_t* iter); -DLL_EXPORT tmq_message_col_t* tmq_get_next_col(tmq_message_tb_t* msg, tmq_col_iter_t* iter); +DLL_EXPORT tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8f49fce558..50af44a94c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -254,6 +254,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) +#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E7) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a8e30bfd52..3b71805e14 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -280,23 +280,21 @@ typedef struct SMqClientTopic { SArray* vgs; //SArray } SMqClientTopic; -typedef struct tmq_resp_err_t { +struct tmq_resp_err_t { int32_t code; -} tmq_resp_err_t; +}; -typedef struct tmq_topic_vgroup_t { +struct tmq_topic_vgroup_t { char* topic; int32_t vgId; int64_t commitOffset; -} tmq_topic_vgroup_t; +}; -typedef struct tmq_topic_vgroup_list_t { +struct tmq_topic_vgroup_list_t { int32_t cnt; int32_t size; tmq_topic_vgroup_t* elems; -} tmq_topic_vgroup_list_t; - -typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); +}; struct tmq_conf_t { char clientId[256]; @@ -677,8 +675,7 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; if (code != 0) { - - printf("exit wait %d\n", pParam->wait); + printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); if (pParam->wait) { tsem_post(&tmq->rspSem); } @@ -769,7 +766,7 @@ END: return 0; } -tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { +tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { int64_t status = atomic_load_64(&tmq->status); tmqAsyncAskEp(tmq, status == 0 || taosArrayGetSize(tmq->clientTopics)); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index eec6a00405..8cca1e6152 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -562,9 +562,9 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } - #endif -TEST(testCase, create_topic_Test) { + +TEST(testCase, create_topic_ctb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -583,13 +583,37 @@ TEST(testCase, create_topic_Test) { taos_free_result(pRes); char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + pRes = taos_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql)); taos_free_result(pRes); taos_close(pConn); } +TEST(testCase, create_topic_stb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -TEST(testCase, tmq_subscribe_Test) { + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + //taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from st1"; + pRes = taos_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +#if 0 +TEST(testCase, tmq_subscribe_ctb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -604,7 +628,32 @@ TEST(testCase, tmq_subscribe_Test) { tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "test_topic_1"); + tmq_list_append(topic_list, "test_ctb_topic_1"); + tmq_subscribe(tmq, topic_list); + + while (1) { + tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + //printf("get msg\n"); + //if (msg == NULL) break; + } +} + +TEST(testCase, tmq_subscribe_stb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); tmq_subscribe(tmq, topic_list); while (1) { @@ -613,6 +662,7 @@ TEST(testCase, tmq_subscribe_Test) { //if (msg == NULL) break; } } +#endif TEST(testCase, tmq_consume_Test) { } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 0c0d8bd349..fa8c78f4b1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -96,7 +96,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { bool found = 0; bool changed = 0; for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { - if (*(int64_t*)taosArrayGet(pSub->availConsumer, j) == consumerId) { + if (*(int64_t *)taosArrayGet(pSub->availConsumer, j) == consumerId) { found = 1; break; } @@ -105,16 +105,13 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { taosArrayPush(pSub->availConsumer, &consumerId); } - int32_t assignedSz = taosArrayGetSize(pSub->assigned); + int32_t assignedSz = taosArrayGetSize(pSub->assigned); topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); for (int32_t j = 0; j < assignedSz; j++) { SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, j); if (pCEp->consumerId == consumerId) { pCEp->lastConsumerHbTs = currentTs; - SMqSubVgEp vgEp = { - .epSet = pCEp->epSet, - .vgId = pCEp->vgId - }; + SMqSubVgEp vgEp = {.epSet = pCEp->epSet, .vgId = pCEp->vgId}; taosArrayPush(topicEp.vgs, &vgEp); changed = 1; } @@ -123,7 +120,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { taosArrayPush(rsp.topics, &topicEp); } if (changed || found) { - SSdbRaw* pRaw = mndSubActionEncode(pSub); + SSdbRaw *pRaw = mndSubActionEncode(pSub); sdbSetRawStatus(pRaw, SDB_STATUS_READY); sdbWriteNotFree(pMnode->pSdb, pRaw); } @@ -137,7 +134,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } void *abuf = buf; tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); - //TODO: free rsp + // TODO: free rsp pMsg->pCont = buf; pMsg->contLen = tlen; return 0; @@ -164,9 +161,9 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { int32_t sz; while (pIter != NULL) { for (int i = 0; i < taosArrayGetSize(pSub->assigned); i++) { - SMqConsumerEp* pCEp = taosArrayGet(pSub->assigned, i); - int64_t consumerId = pCEp->consumerId; - if(pCEp->lastConsumerHbTs != -1 && currentTs - pCEp->lastConsumerHbTs > MND_SUBSCRIBE_REBALANCE_MS) { + SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); + int64_t consumerId = pCEp->consumerId; + if (pCEp->lastConsumerHbTs != -1 && currentTs - pCEp->lastConsumerHbTs > MND_SUBSCRIBE_REBALANCE_MS) { // put consumer into lostConsumer taosArrayPush(pSub->lostConsumer, pCEp); // put vg into unassgined @@ -176,7 +173,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { taosArrayRemove(pSub->assigned, i); // remove from available consumer for (int j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { - if (*(int64_t*)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) { + if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) { taosArrayRemove(pSub->availConsumer, j); break; } @@ -209,7 +206,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { taosArrayPush(pSub->assigned, pCEp); pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); - SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); pConsumer->epoch++; /*SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ @@ -269,33 +266,50 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { // convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); - SArray *pArray; + SArray *pArray = NULL; SArray *inner = taosArrayGet(pDag->pSubplans, 0); SSubplan *plan = taosArrayGetP(inner, 0); + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; - plan->execNode.nodeId = 2; - SEpSet* pEpSet = &plan->execNode.epset; + void *pIter = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pTopic->dbUid) continue; + + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup); + + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql); + return -1; + } + if (pArray && taosArrayGetSize(pArray) != 1) { + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + mError("unsupport topic: %s, sql: %s, plan level: %ld", pTopic->name, pTopic->sql, taosArrayGetSize(pArray)); + return -1; + } - pEpSet->inUse = 0; - addEpIntoEpSet(pEpSet, "localhost", 6030); - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - return -1; - } - int32_t sz = taosArrayGetSize(pArray); - // convert dag to msg - for (int32_t i = 0; i < sz; i++) { SMqConsumerEp CEp; CEp.status = 0; CEp.consumerId = -1; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; - STaskInfo *pTaskInfo = taosArrayGet(pArray, i); + STaskInfo *pTaskInfo = taosArrayGet(pArray, 0); CEp.epSet = pTaskInfo->addr.epset; - - /*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], - * CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/ CEp.vgId = pTaskInfo->addr.nodeId; + + ASSERT(CEp.vgId == pVgroup->vgId); CEp.qmsg = strdup(pTaskInfo->msg->msg); taosArrayPush(unassignedVg, &CEp); + //TODO: free taskInfo + taosArrayDestroy(pArray); + + /*SEpSet *pEpSet = &plan->execNode.epset;*/ + /*pEpSet->inUse = 0;*/ + /*addEpIntoEpSet(pEpSet, "localhost", 6030);*/ + } /*qDestroyQueryDag(pDag);*/ @@ -608,7 +622,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); - bool create = false; + bool create = false; if (pSub == NULL) { mDebug("create new subscription, group: %s, topic %s", consumerGroup, newTopicName); pSub = tNewSubscribeObj(); @@ -624,7 +638,10 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { strcpy(pSub->key, key); free(key); // set unassigned vg - mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); + if (mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg) < 0) { + //TODO: free memory + return -1; + } // TODO: disable alter create = true; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d7fd02422b..4e5246b0a8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -866,18 +866,16 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { if (pHandle->pBlock == NULL) return false; pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); - if (pHandle->tbUid == pHandle->pBlock->uid) { + /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ + ASSERT(pHandle->tbIdHash); + void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); + if (ret != NULL) { pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); return true; - } else if (pHandle->tbIdHash != NULL) { - void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); - if (ret != NULL) { - return true; - } } } return false; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7b1ef01fb2..e2e2e7c8d5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5477,7 +5477,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); - tqReadHandleSetTbUid(streamReadHandle, pTableIdList); + tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); pInfo->readerHandle = streamReadHandle; @@ -7776,7 +7776,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. STableGroupInfo groupInfo = {0}; - int32_t code = doCreateTableGroup(((STqReadHandle*)readerHandle)->pMeta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(((STqReadHandle*)readerHandle)->pVnodeMeta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 489fff5d64..ee5bea0ab7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -254,6 +254,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retriev TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") +// mnode-topic +TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") From afd1ce63db59bc91a83ce06482b29312fc01a5ba Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Jan 2022 10:44:02 +0800 Subject: [PATCH 18/42] [td-11818] Refactor API. --- include/libs/executor/executor.h | 14 +- include/libs/function/function.h | 30 +-- include/libs/function/functionMgt.h | 8 +- include/libs/planner/plannerOp.h | 3 +- source/dnode/vnode/inc/tsdb.h | 22 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 46 ++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 8 +- source/libs/executor/inc/executorimpl.h | 20 +- source/libs/executor/src/executor.c | 4 +- source/libs/executor/src/executorMain.c | 20 +- source/libs/executor/src/executorimpl.c | 232 +++++++++------------ source/libs/function/inc/taggfunction.h | 2 +- source/libs/function/src/taggfunction.c | 218 +++++++++---------- source/libs/function/src/tudf.c | 2 +- source/libs/planner/src/physicalPlan.c | 2 +- source/libs/planner/src/physicalPlanJson.c | 3 - 17 files changed, 297 insertions(+), 339 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 26733696d2..d9e1957e5d 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -27,6 +27,10 @@ typedef void* DataSinkHandle; struct SRpcMsg; struct SSubplan; +typedef struct SReadHandle { + void* reader; + void* meta; +} SReadHandle; /** * Create the exec task for streaming mode * @param pMsg @@ -35,7 +39,13 @@ struct SSubplan; */ qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); -int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); +/** + * + * @param tinfo + * @param input + * @return + */ +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); /** * Create the exec task object according to task json @@ -46,7 +56,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); * @param qId * @return */ -int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, diff --git a/include/libs/function/function.h b/include/libs/function/function.h index b15d5ad33a..aef5f7fec4 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -107,14 +107,14 @@ typedef struct SPoint1 { union{double val; char* ptr;}; } SPoint1; -struct SQLFunctionCtx; +struct SqlFunctionCtx; struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SExtTagsInfo { int16_t tagsLen; // keep the tags data for top/bottom query result int16_t numOfTagCols; - struct SQLFunctionCtx **pTagCtxList; + struct SqlFunctionCtx **pTagCtxList; } SExtTagsInfo; typedef struct SResultDataInfo { @@ -126,18 +126,18 @@ typedef struct SResultDataInfo { #define GET_RES_INFO(ctx) ((ctx)->resultInfo) typedef struct SFunctionFpSet { - bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(struct SQLFunctionCtx *pCtx); + bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(struct SqlFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(struct SQLFunctionCtx *pCtx); - void (*combine)(struct SQLFunctionCtx *pCtx); + void (*finalize)(struct SqlFunctionCtx *pCtx); + void (*combine)(struct SqlFunctionCtx *pCtx); } SFunctionFpSet; extern SFunctionFpSet fpSet[1]; // sql function runtime context -typedef struct SQLFunctionCtx { +typedef struct SqlFunctionCtx { int32_t size; // number of rows void * pInput; // input data buffer uint32_t order; // asc|desc @@ -167,7 +167,7 @@ typedef struct SQLFunctionCtx { int32_t columnIndex; SFunctionFpSet* fpSet; -} SQLFunctionCtx; +} SqlFunctionCtx; enum { TEXPR_NODE_DUMMY = 0x0, @@ -216,14 +216,14 @@ typedef struct SAggFunctionInfo { int8_t sFunctionId; // Transfer function for super table query uint16_t status; - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SQLFunctionCtx *pCtx); + bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(SqlFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SQLFunctionCtx *pCtx); - void (*combine)(SQLFunctionCtx *pCtx); + void (*finalize)(SqlFunctionCtx *pCtx); + void (*combine)(SqlFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); + int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; struct SScalarFuncParam; @@ -279,9 +279,9 @@ void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc); tExprNode* exprdup(tExprNode* pTree); -void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num); +void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); -int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num); +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 3c365335be..aba647f5cd 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -22,7 +22,7 @@ extern "C" { #include "nodes.h" -struct SQLFunctionCtx; +struct SqlFunctionCtx; struct SResultRowEntryInfo; struct STimeWindow; @@ -32,9 +32,9 @@ typedef struct SFuncExecEnv { typedef void* FuncMgtHandle; typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv); -typedef bool (*FExecInit)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); -typedef void (*FExecProcess)(struct SQLFunctionCtx *pCtx); -typedef void (*FExecFinalize)(struct SQLFunctionCtx *pCtx); +typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx); +typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); typedef struct SFuncExecFuncs { FExecGetEnv getEnv; diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 42d3307ac8..31f5457c90 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -24,7 +24,6 @@ #endif OP_ENUM_MACRO(StreamScan) -OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) @@ -48,3 +47,5 @@ OP_ENUM_MACRO(AllTimeWindow) OP_ENUM_MACRO(AllMultiTableTimeInterval) OP_ENUM_MACRO(Order) OP_ENUM_MACRO(Exchange) + +//OP_ENUM_MACRO(TableScan) diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index e43e843218..a2f935683e 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -91,7 +91,7 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); -typedef void* tsdbReadHandleT; +typedef void* tsdbReaderT; /** * Get the data block iterator, starting from position according to the query condition @@ -103,7 +103,7 @@ typedef void* tsdbReadHandleT; * @param qinfo query info handle from query processor * @return */ -tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); +tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -115,13 +115,13 @@ tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup * @param tableInfo table list. * @return */ -//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, // SMemRef *pRef); -tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); -bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); +bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); /** * @@ -148,7 +148,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch * @return row size */ -int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle); /** * move to next block if exists @@ -156,7 +156,7 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); * @param pTsdbReadHandle * @return */ -bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); +bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); /** * Get current data block information @@ -165,7 +165,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); +void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); /** * @@ -177,7 +177,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis); /** * @@ -189,7 +189,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataS * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); +SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); /** * destroy the created table group list, which is generated by tag query @@ -220,7 +220,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGro * clean up the query handle * @param queryHandle */ -void tsdbCleanupReadHandle(tsdbReadHandleT queryHandle); +void tsdbCleanupReadHandle(tsdbReaderT queryHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ac5bde7af2..75afdcfd47 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -71,7 +71,7 @@ typedef struct { typedef struct STqReadHandle { int64_t ver; uint64_t tbUid; - SSubmitMsg* pMsg; + const SSubmitMsg* pMsg; SSubmitBlk* pBlock; SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8965e16e40..1351b9627b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -159,7 +159,7 @@ static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGro static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); //static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void changeQueryHandleForInterpQuery(tsdbReaderT pHandle); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); @@ -167,7 +167,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); //static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); //static void* doFreeColumnInfoData(SArray* pColumnInfoData); //static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); +static bool tsdbGetExternalRow(tsdbReaderT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -208,7 +208,7 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load return pLocalIdList; } -//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { // STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // // int64_t rows = 0; @@ -424,7 +424,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (tsdbReadHandleT)pReadHandle; + return (tsdbReaderT)pReadHandle; _end: tsdbCleanupReadHandle(pReadHandle); @@ -432,14 +432,14 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, return NULL; } -tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } if (emptyQueryTimewindow(pTsdbReadHandle)) { - return (tsdbReadHandleT*) pTsdbReadHandle; + return (tsdbReaderT*) pTsdbReadHandle; } // todo apply the lastkey of table check to avoid to load header file @@ -453,10 +453,10 @@ tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroup tsdbDebug("%p total numOfTable:%" PRIzu " in this query, group %"PRIzu" %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(groupList->pGroupList), pTsdbReadHandle->idStr); - return (tsdbReadHandleT) pTsdbReadHandle; + return (tsdbReaderT) pTsdbReadHandle; } -void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { +void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond *pCond) { STsdbReadHandle* pTsdbReadHandle = queryHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -493,7 +493,7 @@ void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { +void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { STsdbReadHandle* pTsdbReadHandle = queryHandle; pTsdbReadHandle->order = pCond->order; @@ -533,7 +533,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -561,7 +561,7 @@ tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup } #if 0 -tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); if (pTsdbReadHandle == NULL) { return NULL; @@ -581,7 +581,7 @@ tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGro } #endif -SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { +SArray* tsdbGetQueriedTableList(tsdbReaderT *pHandle) { assert(pHandle != NULL); STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; @@ -624,7 +624,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -2338,7 +2338,7 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { cur->blockCompleted = false; } #if 0 -int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; @@ -2494,7 +2494,7 @@ static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) { // filter the queried time stamp in the first place STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; @@ -2635,7 +2635,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { } if (exists) { - tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + tsdbRetrieveDataBlock((tsdbReaderT*) pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); @@ -2891,7 +2891,7 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { } // handle data in cache situation -bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { +bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -3055,7 +3055,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { // return terrno; //} -bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { +bool tsdbGetExternalRow(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; SQueryFilePos* cur = &pTsdbReadHandle->cur; @@ -3112,7 +3112,7 @@ bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { // return code; //} -bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { +bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle) { return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } @@ -3230,7 +3230,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { +void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; @@ -3254,7 +3254,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStatis** pBlockStatis) { STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; @@ -3309,7 +3309,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataS return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range @@ -3820,7 +3820,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { } -void tsdbCleanupReadHandle(tsdbReadHandleT queryHandle) { +void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; if (pTsdbReadHandle == NULL) { return; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6692264c54..21f5d027ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -26,12 +26,14 @@ int vnodeQueryOpen(SVnode *pVnode) { int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in query queue is processing"); + SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta}; switch (pMsg->msgType) { - case TDMT_VND_QUERY: - return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_QUERY:{ + return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); + } case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7af2e9f14f..41ed1739b9 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -224,12 +224,12 @@ typedef struct STaskAttr { // SFilterInfo *pFilters; void* tsdb; -// SMemRef memRef; STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; SArray *pUdfInfo; // no need to free } STaskAttr; +typedef int32_t (*__optr_prepare_fn_t)(void* param); typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); @@ -313,8 +313,9 @@ typedef struct SOperatorInfo { struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + __optr_prepare_fn_t prepareFn; __operator_fn_t exec; - __optr_cleanup_fn_t cleanup; + __optr_cleanup_fn_t cleanupFn; } SOperatorInfo; enum { @@ -395,7 +396,7 @@ typedef struct STableScanInfo { int32_t current; int32_t reverseTimes; // 0 by default - SQLFunctionCtx *pCtx; // next operator query context + SqlFunctionCtx *pCtx; // next operator query context SResultRowInfo *pResultRowInfo; int32_t *rowCellInfoOffset; SExprInfo *pExpr; @@ -425,7 +426,7 @@ typedef struct SStreamBlockScanInfo { typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; int32_t *rowCellInfoOffset; // offset value for each row result cell info - SQLFunctionCtx *pCtx; + SqlFunctionCtx *pCtx; SSDataBlock *pRes; } SOptrBasicInfo; @@ -564,7 +565,6 @@ typedef struct SOrderOperatorInfo { SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); @@ -607,11 +607,11 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); +void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); void freeParam(STaskParam *param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, @@ -659,8 +659,8 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId); +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a32268bf47..c47c83ba29 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -39,7 +39,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } } -int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; } @@ -50,7 +50,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo)); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*) input, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 8635b2d5c8..f632a84c1f 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -51,25 +51,7 @@ static void freeqinfoFn(void *qhandle) { qDestroyTask(*handle); } -void freeParam(STaskParam *param) { - tfree(param->sql); - tfree(param->tagCond); - tfree(param->tbnameCond); - tfree(param->pTableIdList); - taosArrayDestroy(param->pOperator); - tfree(param->pExprs); - tfree(param->pSecExprs); - - tfree(param->pExpr); - tfree(param->pSecExpr); - - tfree(param->pGroupColIndex); - tfree(param->pTagColumnInfo); - tfree(param->pGroupbyExpr); - tfree(param->prevResult); -} - -int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 665f4bc29c..e84fe3d45a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -173,13 +173,13 @@ static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, +static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset); -void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx); -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); +static void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static bool hasMainOutput(STaskAttr *pQueryAttr); @@ -219,14 +219,14 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); -static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); static void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); + SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); SArray* getOrderCheckColumns(STaskAttr* pQuery); @@ -354,7 +354,7 @@ void* destroyOutputBuf(SSDataBlock* pBlock) { return NULL; } -static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static bool isSelectivityWithTagsQuery(SqlFunctionCtx *pCtx, int32_t numOfOutput) { return true; // bool hasTags = false; // int32_t numOfSelectivity = 0; @@ -754,14 +754,14 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf } static bool chkWindowOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + bool masterscan, SResultRow **pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -940,7 +940,7 @@ static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlock return num; } -static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, +static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; bool hasAggregates = pCtx[0].isAggSet; @@ -1073,7 +1073,7 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr *pQueryAttr, STimeWindow *p return ekey; } -static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { +static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { if (type == RESULT_ROW_START_INTERP) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].start.key = INT64_MIN; @@ -1112,9 +1112,9 @@ static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* return ts; } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1124,7 +1124,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { // if (pCtx[0].functionId == FUNCTION_ARITHM) { // SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; // if (pSupport->colList == NULL) { @@ -1141,7 +1141,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo // } } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1205,7 +1205,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, } } -static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { if (functionNeedToExecute(NULL, &pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct @@ -1214,7 +1214,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx, int32_t numOfOutput) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1242,7 +1242,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SExprInfo* pExpr = pOperator->pExpr; - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { int32_t functionId = pCtx[k].functionId; @@ -1302,7 +1302,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } } -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1333,7 +1333,7 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLF return true; } -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { STaskRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1365,7 +1365,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFun return true; } -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1775,7 +1775,7 @@ static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicI int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; - SQLFunctionCtx *pCtx = binfo->pCtx; + SqlFunctionCtx *pCtx = binfo->pCtx; // not assign result buffer yet, add new result buffer, TODO remove it char* d = pData; @@ -1824,7 +1824,7 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // in case of timestamp column, always generated results. @@ -1855,7 +1855,7 @@ static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx * return true; } -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { +void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { SColumnDataAgg *pAgg = NULL; if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColumn->flag)) { @@ -1879,7 +1879,7 @@ void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* } // set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { return TSDB_CODE_SUCCESS; } @@ -1887,8 +1887,8 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { int32_t num = 0; int16_t tagLen = 0; - SQLFunctionCtx* p = NULL; - SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + SqlFunctionCtx* p = NULL; + SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); if (pTagCtx == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1920,11 +1920,11 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return TSDB_CODE_SUCCESS; } -static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } @@ -1937,7 +1937,7 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI for (int32_t i = 0; i < numOfOutput; ++i) { SSqlExpr *pSqlExpr = &pExpr[i].base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; + SqlFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2024,10 +2024,10 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { +static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { size_t numOfOutput = taosArrayGetSize(pExprInfo); - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } @@ -2042,7 +2042,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); SSqlExpr *pSqlExpr = &pExpr->base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; + SqlFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2128,7 +2128,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC return pFuncCtx; } -static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -2773,7 +2773,7 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SqlFunctionCtx *pCtx, int32_t numOfRows) { // STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { @@ -3053,7 +3053,7 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { - SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; uint32_t status = BLK_DATA_NO_NEEDED; int32_t numOfOutput = pTableScanInfo->numOfOutput; @@ -3334,7 +3334,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } /* - * set tag value in SQLFunctionCtx + * set tag value in SqlFunctionCtx * e.g.,tag information into input buffer */ static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes) { @@ -3375,7 +3375,7 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num return NULL; } -void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SExprInfo *pExpr = pOperatorInfo->pExpr; @@ -3509,7 +3509,7 @@ static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { } -void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); } @@ -3531,7 +3531,7 @@ int32_t initResultRow(SResultRow *pResultRow) { * offset[0] offset[1] offset[2] */ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; @@ -3568,7 +3568,7 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t stage, SExecTaskInfo* pTaskInfo) { SOptrBasicInfo *pInfo = &pAggInfo->binfo; - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; @@ -3637,7 +3637,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf } } -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) { bool needCopyTs = false; int32_t tsNum = 0; char *src = NULL; @@ -3680,7 +3680,7 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { } } -void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { +void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (isRowEntryInitialized(pResInfo)) { @@ -3701,7 +3701,7 @@ void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { } } -static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { // if (pRuntimeEnv->pTsBuf) { // SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); // bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); @@ -3722,7 +3722,7 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt pTableScanInfo->reverseTimes = 0; } -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -3832,7 +3832,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3865,7 +3865,7 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pRes } } -void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, +void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -3905,7 +3905,7 @@ void setExecutionContext(STaskRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, in pRuntimeEnv->prevGroupId = tableGroupId; } -void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3928,7 +3928,7 @@ void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFu } } -void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { +void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SSqlExpr* pExpr = &pExprInfo->base; @@ -3996,7 +3996,7 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S } // TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { #if 0 STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4031,7 +4031,7 @@ void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, #endif } -void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { +void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 int32_t numOfExprs = pQueryAttr->numOfOutput; @@ -4199,7 +4199,7 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntim } } -static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, +static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4953,6 +4953,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->pTsdbReadHandle == NULL) { return NULL; } @@ -5341,37 +5342,6 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { - assert(repeatTime > 0 && numOfOutput > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pInfo->pTsdbReadHandle = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - - pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->exec = doTableScan; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; -} - SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); @@ -5446,8 +5416,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; +// pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; pOperator->exec = doBlockInfoScan; return pOperator; @@ -5700,9 +5669,8 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - // pOperator->exec = doGlobalAggregate; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->cleanupFn = destroyGlobalAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -5745,7 +5713,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; // pOperator->exec = doMultiwayMergeSort; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->cleanupFn = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5849,7 +5817,7 @@ SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->exec = doSort; - pOperator->cleanup = destroyOrderOperatorInfo; + pOperator->cleanupFn = destroyOrderOperatorInfo; pOperator->pRuntimeEnv = pRuntimeEnv; appendDownstream(pOperator, downstream); @@ -6758,8 +6726,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->cleanup != NULL) { - pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + if (pOperator->cleanupFn != NULL) { + pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -6813,7 +6781,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; - pOperator->cleanup = destroyAggOperatorInfo; + pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6908,7 +6876,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableAggregate; - pOperator->cleanup = destroyAggOperatorInfo; + pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6938,7 +6906,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doProjectOperation; - pOperator->cleanup = destroyProjectOperatorInfo; + pOperator->cleanupFn = destroyProjectOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6996,7 +6964,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->exec = doFilter; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyConditionOperatorInfo; + pOperator->cleanupFn = destroyConditionOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7038,7 +7006,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7063,7 +7031,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAllIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7087,7 +7055,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doStateWindowAgg; - pOperator->cleanup = destroyStateWindowOperatorInfo; + pOperator->cleanupFn = destroyStateWindowOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7112,7 +7080,7 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSessionWindowAgg; - pOperator->cleanup = destroySWindowOperatorInfo; + pOperator->cleanupFn = destroySWindowOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7136,7 +7104,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7160,7 +7128,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); @@ -7193,7 +7161,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashGroupbyAggregate; - pOperator->cleanup = destroyGroupbyOperatorInfo; + pOperator->cleanupFn = destroyGroupbyOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7232,7 +7200,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doFill; - pOperator->cleanup = destroySFillOperatorInfo; + pOperator->cleanupFn = destroySFillOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7280,7 +7248,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI // pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroySlimitOperatorInfo; + pOperator->cleanupFn = destroySlimitOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7438,7 +7406,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyTagScanOperatorInfo; + pOperator->cleanupFn = destroyTagScanOperatorInfo; return pOperator; } @@ -7578,7 +7546,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashDistinct; pOperator->pExpr = pExpr; - pOperator->cleanup = destroyDistinctOperatorInfo; + pOperator->cleanupFn = destroyDistinctOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7748,26 +7716,18 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { return pTaskInfo; } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId); -static int32_t doCreateTableGroup(void* readerHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); +static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId); +static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); -SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { - if (pPhyNode->info.type == OP_TableScan) { - + if (pPhyNode->info.type == OP_DataBlocksOptScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhyNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId); - return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); - } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); - - return createDataBlocksOptScanInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + return createDataBlocksOptScanInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); @@ -7775,10 +7735,11 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. STableGroupInfo groupInfo = {0}; - int32_t code = doCreateTableGroup(((STqReadHandle*)readerHandle)->pMeta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); + // Transfer the Array of STableKeyInfo into uid list. size_t numOfTables = taosArrayGetSize(pa); SArray* idList = taosArrayInit(numOfTables, sizeof(uint64_t)); @@ -7787,7 +7748,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask taosArrayPush(idList, &pkeyInfo->uid); } - SOperatorInfo* pOperator = createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, idList, pTaskInfo); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pPhyNode->pTargets, idList, pTaskInfo); taosArrayDestroy(idList); //TODO destroy groupInfo @@ -7801,18 +7762,23 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask for (int32_t i = 0; i < size; ++i) { SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); - SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId, taskId); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId); return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); } } } -static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { +static tsdbReaderT createDataReaderImpl(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readHandle, uint64_t queryId, uint64_t taskId) { STsdbQueryCond cond = {.loadExternalRows = false}; cond.order = pTableScanNode->scan.order; cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + if (cond.colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + cond.twindow = pTableScanNode->window; cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; @@ -7826,7 +7792,7 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S cond.colList[i].colId = pSchema->colId; } - return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId); + return tsdbQueryTables(readHandle, &cond, pGroupInfo, queryId, taskId); } static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { @@ -7840,11 +7806,11 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t return code; } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId) { +static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { STableGroupInfo groupInfo = {0}; uint64_t uid = pTableScanNode->scan.uid; - int32_t code = doCreateTableGroup(readerHandle, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -7855,14 +7821,14 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, goto _error; } - return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId, taskId); + return createDataReaderImpl(pTableScanNode, &groupInfo, pHandle->reader, queryId, taskId); _error: terrno = code; return NULL; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId) { +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -7872,7 +7838,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* goto _complete; } - (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId, taskId); + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId); if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 8de66584c6..8b83dbe8cd 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -73,7 +73,7 @@ typedef struct STwaInfo { extern int32_t functionCompatList[]; // compatible check array list -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); +bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** * the numOfRes should be kept, since it may be used later diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index ff2fb95328..694a1c2ca5 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -59,7 +59,7 @@ #define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ __ctx->tag.i = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ @@ -71,14 +71,14 @@ #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ aggFunc[FUNCTION_TAG].addInput(__ctx); \ } \ } while (0); -void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop1(SqlFunctionCtx *UNUSED_PARAM(pCtx)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } +void doFinalizer(SqlFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } typedef struct tValuePair { SVariant v; @@ -200,7 +200,7 @@ void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { pCell->initialized = false; } -int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num) { int32_t maxOutput = 0; for (int32_t j = 0; j < num; ++j) { int32_t id = pCtx[j].functionId; @@ -223,7 +223,7 @@ int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { return maxOutput; } -void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num) { +void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num) { for (int32_t j = 0; j < num; ++j) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); pResInfo->numOfRes = 0; @@ -473,7 +473,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; } @@ -490,7 +490,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf * * @param pCtx */ -static void function_finalizer(SQLFunctionCtx *pCtx) { +static void function_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); @@ -503,7 +503,7 @@ static void function_finalizer(SQLFunctionCtx *pCtx) { * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ -static void count_function(SQLFunctionCtx *pCtx) { +static void count_function(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; /* @@ -537,7 +537,7 @@ static void count_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void count_func_merge(SQLFunctionCtx *pCtx) { +static void count_func_merge(SqlFunctionCtx *pCtx) { int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->pOutput) += pData[i]; @@ -555,7 +555,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +int32_t countRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { return BLK_DATA_NO_NEEDED; } else { @@ -563,7 +563,7 @@ int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { } } -int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_NO_NEEDED; } #define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ @@ -635,7 +635,7 @@ int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ } while (0) -static void do_sum(SQLFunctionCtx *pCtx) { +static void do_sum(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; // Only the pre-computing information loaded and actual data does not loaded @@ -698,7 +698,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { } } -static void sum_function(SQLFunctionCtx *pCtx) { +static void sum_function(SqlFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer @@ -710,7 +710,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { } } -static void sum_func_merge(SQLFunctionCtx *pCtx) { +static void sum_func_merge(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); @@ -742,16 +742,16 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } } -static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_STATIS_NEEDED; } -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t dataBlockRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_ALL_NEEDED; } // todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -764,7 +764,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t c } } -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i) { return BLK_DATA_NO_NEEDED; } @@ -776,7 +776,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t co } } -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -796,7 +796,7 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32 } } -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i) { return BLK_DATA_NO_NEEDED; } @@ -822,7 +822,7 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_ * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate * result into output buffer. */ -static void avg_function(SQLFunctionCtx *pCtx) { +static void avg_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf @@ -885,7 +885,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } } -static void avg_func_merge(SQLFunctionCtx *pCtx) { +static void avg_func_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->pOutput; @@ -907,7 +907,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { /* * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ -static void avg_finalizer(SQLFunctionCtx *pCtx) { +static void avg_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { @@ -938,7 +938,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// -static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { +static void minMax_function(SqlFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { // data in current data block are qualified to the query if (pCtx->isAggSet) { *notNullElems = pCtx->size - pCtx->agg.numOfNull; @@ -994,7 +994,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*data < val) ^ isMin) { *data = (int32_t)val; for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -1089,7 +1089,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } } -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1135,7 +1135,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf return true; } -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1184,7 +1184,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf /* * the output result of min/max function is the final output buffer, not the intermediate result buffer */ -static void min_function(SQLFunctionCtx *pCtx) { +static void min_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); @@ -1201,7 +1201,7 @@ static void min_function(SQLFunctionCtx *pCtx) { } } -static void max_function(SQLFunctionCtx *pCtx) { +static void max_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); @@ -1218,7 +1218,7 @@ static void max_function(SQLFunctionCtx *pCtx) { } } -static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { +static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); @@ -1247,7 +1247,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp *(int32_t *)output = v; for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; + SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; aggFunc[FUNCTION_TAG].addInput(__ctx); } @@ -1303,7 +1303,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp return notNullElems; } -static void min_func_merge(SQLFunctionCtx *pCtx) { +static void min_func_merge(SqlFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1314,7 +1314,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { } } -static void max_func_merge(SQLFunctionCtx *pCtx) { +static void max_func_merge(SqlFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1334,7 +1334,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { (r) += TPOW2(((type *)d)[i] - (delta)); \ } -static void stddev_function(SQLFunctionCtx *pCtx) { +static void stddev_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); @@ -1419,7 +1419,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { } } -static void stddev_finalizer(SQLFunctionCtx *pCtx) { +static void stddev_finalizer(SqlFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { @@ -1445,7 +1445,7 @@ int32_t tsCompare(const void* p1, const void* p2) { } } -static void stddev_dst_function(SQLFunctionCtx *pCtx) { +static void stddev_dst_function(SqlFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // the second stage to calculate standard deviation @@ -1536,7 +1536,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); } -static void stddev_dst_merge(SQLFunctionCtx *pCtx) { +static void stddev_dst_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -1553,7 +1553,7 @@ static void stddev_dst_merge(SQLFunctionCtx *pCtx) { } } -static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { +static void stddev_dst_finalizer(SqlFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { @@ -1568,7 +1568,7 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -1581,7 +1581,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* } // todo opt for null block -static void first_function(SQLFunctionCtx *pCtx) { +static void first_function(SqlFunctionCtx *pCtx) { if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1612,7 +1612,7 @@ static void first_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); @@ -1630,7 +1630,7 @@ static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t in * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) * to decide if the value is earlier than current intermediate result */ -static void first_dist_function(SQLFunctionCtx *pCtx) { +static void first_dist_function(SqlFunctionCtx *pCtx) { /* * do not to check data in the following cases: * 1. data block that are not loaded @@ -1661,7 +1661,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_dist_func_merge(SQLFunctionCtx *pCtx) { +static void first_dist_func_merge(SqlFunctionCtx *pCtx) { assert(pCtx->stableQuery); char * pData = GET_INPUT_DATA_LIST(pCtx); @@ -1691,7 +1691,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at * least one data in this block that is not null.(TODO opt for this case) */ -static void last_function(SQLFunctionCtx *pCtx) { +static void last_function(SqlFunctionCtx *pCtx) { if (pCtx->order != pCtx->param[0].i) { return; } @@ -1743,7 +1743,7 @@ static void last_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); @@ -1761,7 +1761,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind } } -static void last_dist_function(SQLFunctionCtx *pCtx) { +static void last_dist_function(SqlFunctionCtx *pCtx) { /* * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data @@ -1796,7 +1796,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { +static void last_dist_func_merge(SqlFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); @@ -1824,7 +1824,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { /* * NOTE: last_row does not use the interResultBuf to keep the result */ -static void last_row_function(SQLFunctionCtx *pCtx) { +static void last_row_function(SqlFunctionCtx *pCtx) { assert(pCtx->size >= 1); char *pData = GET_INPUT_DATA_LIST(pCtx); @@ -1849,7 +1849,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); } -static void last_row_finalizer(SQLFunctionCtx *pCtx) { +static void last_row_finalizer(SqlFunctionCtx *pCtx) { // do nothing at the first stage SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { @@ -1873,7 +1873,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; + SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; if (ctx->functionId == FUNCTION_TS_DUMMY) { ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; ctx->tag.i = tsKey; @@ -2023,7 +2023,7 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } -static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { +static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -2118,7 +2118,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * * top/bottom use the intermediate result buffer to keep the intermediate result */ -static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { +static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer @@ -2136,7 +2136,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { +static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i; @@ -2150,7 +2150,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { } } -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { +bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; @@ -2206,7 +2206,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha } } -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool top_bottom_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2216,7 +2216,7 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* return true; } -static void top_function(SQLFunctionCtx *pCtx) { +static void top_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; STopBotInfo *pRes = getTopBotOutputInfo(pCtx); @@ -2252,7 +2252,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } } -static void top_func_merge(SQLFunctionCtx *pCtx) { +static void top_func_merge(SqlFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data @@ -2275,7 +2275,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { } } -static void bottom_function(SQLFunctionCtx *pCtx) { +static void bottom_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; STopBotInfo *pRes = getTopBotOutputInfo(pCtx); @@ -2309,7 +2309,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } } -static void bottom_func_merge(SQLFunctionCtx *pCtx) { +static void bottom_func_merge(SqlFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data @@ -2332,7 +2332,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { } } -static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { +static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results @@ -2361,7 +2361,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { } /////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool percentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2375,7 +2375,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* return true; } -static void percentile_function(SQLFunctionCtx *pCtx) { +static void percentile_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -2460,7 +2460,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void percentile_finalizer(SQLFunctionCtx *pCtx) { +static void percentile_finalizer(SqlFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -2484,7 +2484,7 @@ static void buildHistogramInfo(SAPercentileInfo* pInfo) { pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); } -static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { +static SAPercentileInfo *getAPerctInfo(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo* pInfo = NULL; @@ -2498,7 +2498,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { return pInfo; } -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool apercentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2510,7 +2510,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo return true; } -static void apercentile_function(SQLFunctionCtx *pCtx) { +static void apercentile_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); @@ -2542,7 +2542,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } } -static void apercentile_func_merge(SQLFunctionCtx *pCtx) { +static void apercentile_func_merge(SqlFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); @@ -2572,7 +2572,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); } -static void apercentile_finalizer(SQLFunctionCtx *pCtx) { +static void apercentile_finalizer(SqlFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); @@ -2608,7 +2608,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2638,7 +2638,7 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInf LEASTSQR_CAL(param, x, y, i, step); \ } -static void leastsquares_function(SQLFunctionCtx *pCtx) { +static void leastsquares_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2724,7 +2724,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { +static void leastsquares_finalizer(SqlFunctionCtx *pCtx) { // no data in query SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2756,12 +2756,12 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void date_col_output_function(SQLFunctionCtx *pCtx) { +static void date_col_output_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); *(int64_t *)(pCtx->pOutput) = pCtx->startTs; } -static void col_project_function(SQLFunctionCtx *pCtx) { +static void col_project_function(SqlFunctionCtx *pCtx) { // the number of output rows should not affect the final number of rows, so set it to be 0 if (pCtx->numOfParams == 2) { return; @@ -2791,7 +2791,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { * @param pCtx * @return */ -static void tag_project_function(SQLFunctionCtx *pCtx) { +static void tag_project_function(SqlFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); @@ -2814,9 +2814,9 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { * @param pCtx * @return */ -static void copy_function(SQLFunctionCtx *pCtx); +static void copy_function(SqlFunctionCtx *pCtx); -static void tag_function(SQLFunctionCtx *pCtx) { +static void tag_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); if (pCtx->currentStage == MERGE_STAGE) { copy_function(pCtx); @@ -2825,7 +2825,7 @@ static void tag_function(SQLFunctionCtx *pCtx) { } } -static void copy_function(SQLFunctionCtx *pCtx) { +static void copy_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); char *pData = GET_INPUT_DATA_LIST(pCtx); @@ -2836,7 +2836,7 @@ enum { INITIAL_VALUE_NOT_ASSIGNED = 0, }; -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2846,7 +2846,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResI return false; } -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2861,7 +2861,7 @@ static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pRes return false; } -static void deriv_function(SQLFunctionCtx *pCtx) { +static void deriv_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3056,7 +3056,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { } while (0); // TODO difference in date column -static void diff_function(SQLFunctionCtx *pCtx) { +static void diff_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); @@ -3236,7 +3236,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } -static void arithmetic_function(SQLFunctionCtx *pCtx) { +static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; @@ -3264,7 +3264,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3283,7 +3283,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pRe return true; } -static void spread_function(SQLFunctionCtx *pCtx) { +static void spread_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3368,7 +3368,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ -void spread_func_merge(SQLFunctionCtx *pCtx) { +void spread_func_merge(SqlFunctionCtx *pCtx) { SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; @@ -3385,7 +3385,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -void spread_function_finalizer(SQLFunctionCtx *pCtx) { +void spread_function_finalizer(SqlFunctionCtx *pCtx) { /* * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary @@ -3423,7 +3423,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * param[2]: end time * @param pCtx */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool twa_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3444,7 +3444,7 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { +static int32_t twa_function_impl(SqlFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3685,7 +3685,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si return notNullElems; } -static void twa_function(SQLFunctionCtx *pCtx) { +static void twa_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3719,7 +3719,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { * by next input data. The TWA function only applies to each table, so no merge procedure * is required, we simply copy to the resut ot interResBuffer. */ -void twa_function_copy(SQLFunctionCtx *pCtx) { +void twa_function_copy(SqlFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3727,7 +3727,7 @@ void twa_function_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } -void twa_function_finalizer(SQLFunctionCtx *pCtx) { +void twa_function_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -3752,7 +3752,7 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { * @param pCtx */ -static void interp_function_impl(SQLFunctionCtx *pCtx) { +static void interp_function_impl(SqlFunctionCtx *pCtx) { int32_t type = (int32_t) pCtx->param[2].i; if (type == TSDB_FILL_NONE) { return; @@ -3875,7 +3875,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); } -static void interp_function(SQLFunctionCtx *pCtx) { +static void interp_function(SqlFunctionCtx *pCtx) { // at this point, the value is existed, return directly if (pCtx->size > 0) { bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); @@ -3918,7 +3918,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool ts_comp_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; // not initialized since it has been initialized } @@ -3929,7 +3929,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pR return true; } -static void ts_comp_function(SQLFunctionCtx *pCtx) { +static void ts_comp_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; @@ -3949,7 +3949,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void ts_comp_finalize(SQLFunctionCtx *pCtx) { +static void ts_comp_finalize(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -4006,7 +4006,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; } -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool rate_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -4023,7 +4023,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResI return true; } -static void rate_function(SQLFunctionCtx *pCtx) { +static void rate_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; @@ -4076,7 +4076,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { } } -static void rate_func_copy(SQLFunctionCtx *pCtx) { +static void rate_func_copy(SqlFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4084,7 +4084,7 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; } -static void rate_finalizer(SQLFunctionCtx *pCtx) { +static void rate_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4102,7 +4102,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void irate_function(SQLFunctionCtx *pCtx) { +static void irate_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; @@ -4184,7 +4184,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi } } -static void blockInfo_func(SQLFunctionCtx* pCtx) { +static void blockInfo_func(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4232,7 +4232,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock } } -void block_func_merge(SQLFunctionCtx* pCtx) { +void block_func_merge(SqlFunctionCtx* pCtx) { STableBlockDist info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); @@ -4338,7 +4338,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { UNUSED(sz); } -void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { +void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4380,12 +4380,12 @@ int32_t functionCompatList[] = { }; //typedef struct SFunctionFpSet { -// bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment -// void (*addInput)(struct SQLFunctionCtx *pCtx); +// bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment +// void (*addInput)(struct SqlFunctionCtx *pCtx); // // // finalizer must be called after all exec has been executed to generated final result. -// void (*finalize)(struct SQLFunctionCtx *pCtx); -// void (*combine)(struct SQLFunctionCtx *pCtx); +// void (*finalize)(struct SqlFunctionCtx *pCtx); +// void (*combine)(struct SqlFunctionCtx *pCtx); //} SFunctionFpSet; SFunctionFpSet fpSet[1] = { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 2771a9ecf8..b65e637a57 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -121,7 +121,7 @@ void destroyUdfInfo(SUdfInfo* pUdfInfo) { tfree(pUdfInfo); } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type) { int32_t output = 0; if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 0be97f1ce0..530d618aee 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -287,7 +287,7 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) { static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) { SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList; vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode); - int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; return createUserTableScanNode(pPlanNode, pTableInfo, type); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 5d96ab47c3..e367f2e74b 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -88,7 +88,6 @@ static const char* jkPnodeType = "Type"; static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { case OP_StreamScan: - case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return sizeof(STableScanPhyNode); @@ -831,7 +830,6 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { const SPhyNode* phyNode = (const SPhyNode*)obj; switch (phyNode->info.type) { case OP_StreamScan: - case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return tableScanNodeToJson(obj, json); @@ -869,7 +867,6 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* phyNode = (SPhyNode*)obj; switch (phyNode->info.type) { - case OP_TableScan: case OP_StreamScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: From 2d439d29e2378bf22a8a3119b478f65be6e38839 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Jan 2022 10:54:50 +0800 Subject: [PATCH 19/42] handle drop stb req --- include/common/tmsg.h | 24 +++++--- source/common/src/tmsg.c | 15 +++++ source/dnode/mgmt/impl/test/vnode/vnode.cpp | 47 ++++++++-------- source/dnode/mnode/impl/src/mndStb.c | 62 +++++++++++---------- 4 files changed, 88 insertions(+), 60 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 27a01e4818..59f175ce83 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -140,6 +140,11 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) + +#define TD_SUPER_TABLE TSDB_SUPER_TABLE +#define TD_CHILD_TABLE TSDB_CHILD_TABLE +#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE + typedef struct { int32_t vgId; char* dbName; @@ -1139,10 +1144,7 @@ typedef struct SVCreateTbReq { char* name; uint32_t ttl; uint32_t keep; -#define TD_SUPER_TABLE TSDB_SUPER_TABLE -#define TD_CHILD_TABLE TSDB_CHILD_TABLE -#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE - uint8_t type; + uint8_t type; union { struct { tb_uid_t suid; @@ -1187,15 +1189,21 @@ typedef struct { } SVAlterTbRsp; typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int64_t suid; + uint64_t ver; + char* name; + uint8_t type; + tb_uid_t suid; } SVDropTbReq; typedef struct { - SMsgHead head; + uint64_t ver; } SVDropTbRsp; +int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); +void* tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq); +int32_t tSerializeSVDropTbRsp(void** buf, SVDropTbRsp* pRsp); +void* tDeserializeSVDropTbRsp(void* buf, SVDropTbRsp* pRsp); + typedef struct { SMsgHead head; int64_t uid; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 48e9dce3c1..4b18d7d9a1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -310,3 +310,18 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { return buf; } + +int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU8(buf, pReq->type); + return tlen; +} + +void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { + buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeString(buf, &pReq->name); + buf = taosDecodeFixedU8(buf, &pReq->type); + return buf; +} diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 8e56c31ed1..11b32fbf0f 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -199,19 +199,16 @@ TEST_F(DndTestVnode, 03_Create_Stb) { req.stbCfg.nTagCols = 3; req.stbCfg.pTagSchema = &schemas[2]; - int32_t bsize = tSerializeSVCreateTbReq(NULL, &req); - void* buf = rpcMallocCont(sizeof(SMsgHead) + bsize); - SMsgHead* pMsgHead = (SMsgHead*)buf; + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(2); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); - void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - int32_t contLen = sizeof(SMsgHead) + bsize; - - SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, buf, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { ASSERT_EQ(pRsp->code, 0); @@ -236,23 +233,25 @@ TEST_F(DndTestVnode, 04_ALTER_Stb) { TEST_F(DndTestVnode, 05_DROP_Stb) { { - int32_t contLen = sizeof(SVDropTbReq); - SVDropTbReq* pReq = (SVDropTbReq*)rpcMallocCont(contLen); - strcpy(pReq->name, "stb1"); - pReq->suid = 0; - - SMsgHead* pMsgHead = (SMsgHead*)&pReq->head; - pMsgHead->contLen = htonl(contLen); - pMsgHead->vgId = htonl(2); - for (int i = 0; i < 3; ++i) { - SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, pReq, contLen); + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char*)"stb1"; + req.suid = 9599; + req.type = TD_SUPER_TABLE; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); + + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - } // else { - // ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); - //} + ASSERT_EQ(pRsp->code, 0); } } } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 450a318b26..111f092dc3 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -231,15 +231,11 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { } static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SVCreateTbReq req = {0}; - void *buf = NULL; - int32_t bsize = 0; - SMsgHead *pMsgHead = NULL; - - req.ver = 0; SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + SVCreateTbReq req = {0}; + req.ver = 0; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; @@ -250,40 +246,48 @@ static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb req.stbCfg.nTagCols = pStb->numOfTags; req.stbCfg.pTagSchema = pStb->pSchema + pStb->numOfColumns; - bsize = tSerializeSVCreateTbReq(NULL, &req); - buf = malloc(sizeof(SMsgHead) + bsize); - if (buf == NULL) { + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pMsgHead = (SMsgHead *)buf; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(pVgroup->vgId); - - void *pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - *pContLen = sizeof(SMsgHead) + bsize; - return buf; + *pContLen = contLen; + return pHead; } -static SVDropTbReq *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SVDropTbReq); +static void *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVDropTbReq *pDrop = calloc(1, contLen); - if (pDrop == NULL) { + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char *)tNameGetTableName(&name); + req.type = TD_SUPER_TABLE; + req.suid = pStb->uid; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pDrop->head.contLen = htonl(contLen); - pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pDrop->suid = htobe64(pStb->uid); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - return pDrop; + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; } static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { @@ -403,7 +407,8 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SVDropTbReq *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb); + int32_t contLen = 0; + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb, &contLen); if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -414,7 +419,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; - action.contLen = sizeof(SVDropTbReq); + action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -625,7 +630,8 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb); + int32_t contLen = 0; + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb, &contLen); if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); From 43b0b23cd34d05026215327aac60def6766f37b5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Jan 2022 11:59:08 +0800 Subject: [PATCH 20/42] [td-11818] Update the physical plan in case of the subscribe processing. --- source/libs/planner/src/physicalPlan.c | 39 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 530d618aee..5d0600c820 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -296,6 +296,7 @@ static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNo if (needMultiNodeScan(pTable)) { return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } + return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan); } @@ -386,6 +387,33 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { // todo deal subquery } +static void postCreateDag(SQueryPlanNode* pQueryNode, SQueryDag* pDag, SArray* pNodeList) { + // The exchange operator is not necessary, in case of the stream scan. + // Here we need to remove it from the DAG. + if (pQueryNode->info.type == QNODE_STREAMSCAN) { + SArray* pRootLevel = taosArrayGetP(pDag->pSubplans, 0); + SSubplan *pSubplan = taosArrayGetP(pRootLevel, 0); + + if (pSubplan->pNode->info.type == OP_Exchange) { + ASSERT(taosArrayGetSize(pRootLevel) == 1); + + taosArrayRemove(pDag->pSubplans, 0); + // And then update the number of the subplans. + pDag->numOfSubplans -= 1; + } + } else { + // Traverse the dag again to acquire the execution node. + if (pNodeList != NULL) { + SArray** pSubLevel = taosArrayGetLast(pDag->pSubplans); + size_t num = taosArrayGetSize(*pSubLevel); + for (int32_t j = 0; j < num; ++j) { + SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); + taosArrayPush(pNodeList, &pPlan->execNode); + } + } + } +} + int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId) { TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { @@ -407,16 +435,7 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_FAILED; } END_TRY - // traverse the dag again to acquire the execution node. - if (pNodeList != NULL) { - SArray** pSubLevel = taosArrayGetLast((*pDag)->pSubplans); - size_t num = taosArrayGetSize(*pSubLevel); - for (int32_t j = 0; j < num; ++j) { - SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); - taosArrayPush(pNodeList, &pPlan->execNode); - } - } - + postCreateDag(pQueryNode, *pDag, pNodeList); return TSDB_CODE_SUCCESS; } From 660a600dd6bc52a6afd8f626f3967b1cd6d1203e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 13:15:41 +0800 Subject: [PATCH 21/42] refactor --- source/client/src/clientImpl.c | 230 +++++++++++++++++++-------------- 1 file changed, 130 insertions(+), 100 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3b71805e14..7e95d55ec7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -21,6 +21,93 @@ } \ } while (0) +typedef struct SMqClientVg { + // statistics + int64_t pollCnt; + // offset + int64_t committedOffset; + int64_t currentOffset; + //connection info + int32_t vgId; + SEpSet epSet; +} SMqClientVg; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; //SArray +} SMqClientTopic; + +struct tmq_resp_err_t { + int32_t code; +}; + +struct tmq_topic_vgroup_t { + char* topic; + int32_t vgId; + int64_t commitOffset; +}; + +struct tmq_topic_vgroup_list_t { + int32_t cnt; + int32_t size; + tmq_topic_vgroup_t* elems; +}; + +typedef struct SMqConsumeCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + tmq_message_t** retMsg; +} SMqConsumeCbParam; + +struct tmq_conf_t { + char clientId[256]; + char groupId[256]; + char* ip; + uint16_t port; + tmq_commit_cb* commit_cb; +}; + +struct tmq_message_t { + SMqConsumeRsp rsp; +}; + + +tmq_conf_t* tmq_conf_new() { + tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + return conf; +} + +int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (strcmp(key, "group.id") == 0) { + strcpy(conf->groupId, value); + } + if (strcmp(key, "client.id") == 0) { + strcpy(conf->clientId, value); + } + return 0; +} + +struct tmq_t { + char groupId[256]; + char clientId[256]; + SRWLatch lock; + int64_t consumerId; + int64_t epoch; + int64_t status; + tsem_t rspSem; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + int32_t nextTopicIdx; + SArray* clientTopics; //SArray + //stat + int64_t pollCnt; +}; + static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); @@ -259,81 +346,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) } -typedef struct SMqClientVg { - // statistics - int64_t pollCnt; - // offset - int64_t committedOffset; - int64_t currentOffset; - //connection info - int32_t vgId; - SEpSet epSet; -} SMqClientVg; - -typedef struct SMqClientTopic { - // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - int32_t nextVgIdx; - SArray* vgs; //SArray -} SMqClientTopic; - -struct tmq_resp_err_t { - int32_t code; -}; - -struct tmq_topic_vgroup_t { - char* topic; - int32_t vgId; - int64_t commitOffset; -}; - -struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; -}; - -struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - char* ip; - uint16_t port; - tmq_commit_cb* commit_cb; -}; - -tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); - return conf; -} - -int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); - } - if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); - } - return 0; -} - -struct tmq_t { - char groupId[256]; - char clientId[256]; - SRWLatch lock; - int64_t consumerId; - int64_t epoch; - int64_t status; - tsem_t rspSem; - STscObj* pTscObj; - tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - SArray* clientTopics; //SArray - //stat - int64_t pollCnt; -}; tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = calloc(sizeof(tmq_t), 1); @@ -580,7 +592,7 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int tlen = tSerializeSCMCreateTopicReq(NULL, &req); void* buf = malloc(tlen); - if(buf == NULL) { + if (buf == NULL) { goto _return; } @@ -614,10 +626,6 @@ _return: /*typedef SMqConsumeRsp tmq_message_t;*/ -struct tmq_message_t { - SMqConsumeRsp rsp; -}; - int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { if (code == -1) { printf("discard\n"); @@ -766,47 +774,69 @@ END: return 0; } +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, SMqClientVg** ppVg) { + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + if (pReq == NULL) { + return NULL; + } + pReq->reqType = type; + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + strcpy(pReq->cgroup, tmq->groupId); + + tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); + strcpy(pReq->topic, pTopic->topicName); + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + pReq->offset = pVg->currentOffset+1; + + pReq->head.vgId = htonl(pVg->vgId); + pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + return pReq; +} + tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* tmq_message = NULL; + int64_t status = atomic_load_64(&tmq->status); tmqAsyncAskEp(tmq, status == 0 || taosArrayGetSize(tmq->clientTopics)); - if (blocking_time < 0) blocking_time = 500; + /*if (blocking_time < 0) blocking_time = 500;*/ + blocking_time = 1000; if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); usleep(blocking_time * 1000); return NULL; } - - SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); - pReq->reqType = 1; - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; - tmq_message_t* tmq_message = NULL; - strcpy(pReq->cgroup, tmq->groupId); - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - strcpy(pReq->topic, pTopic->topicName); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); - if (vgSz == 0) { - free(pReq); + if (taosArrayGetSize(pTopic->vgs) == 0) { usleep(blocking_time * 1000); return NULL; } - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % vgSz); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - pReq->offset = pVg->currentOffset+1; - pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + SMqClientVg* pVg = NULL; + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, &pVg); + if (pReq == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + + SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + if (param == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + param->tmq = tmq; + param->retMsg = &tmq_message; + param->pVg = pVg; SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; - sendInfo->param = pVg; + sendInfo->param = param; sendInfo->fp = tmq_poll_cb_inner; /*printf("req offset: %ld\n", pReq->offset);*/ From c2717371ab6d1db4605e961c71f4cd5e48d423ed Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 13:16:07 +0800 Subject: [PATCH 22/42] refactor --- include/client/taos.h | 1 + include/common/common.h | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/include/client/taos.h b/include/client/taos.h index 5f7cd12963..6549821953 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -225,6 +225,7 @@ DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); DLL_EXPORT tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time); +DLL_EXPORT tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async); #ifdef __cplusplus } diff --git a/include/common/common.h b/include/common/common.h index b1231be5ae..bbde65b48f 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -38,6 +38,10 @@ // int16_t bytes; //} SSchema; +#define TMQ_REQ_TYPE_COMMIT_ONLY 0 +#define TMQ_REQ_TYPE_CONSUME_ONLY 1 +#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 + typedef struct { uint32_t numOfTables; SArray *pGroupList; From 6c6573cc4d15bb7da041a4bbd019c57702c89b97 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 28 Jan 2022 13:44:40 +0800 Subject: [PATCH 23/42] optimize trans --- source/libs/transport/inc/transComm.h | 12 +++++++++ source/libs/transport/src/transCli.c | 26 ++++++++++++-------- source/libs/transport/src/transComm.c | 32 ++++++++++++++++++++++++ source/libs/transport/src/transSrv.c | 31 ++++++++++++------------ source/libs/transport/test/rclient.c | 35 +++++++++++++++++++++------ 5 files changed, 103 insertions(+), 33 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 3f2aa1170e..f4fe0b1f79 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -211,6 +211,18 @@ typedef struct SConnBuffer { int left; } SConnBuffer; +typedef void (*AsyncCB)(uv_async_t* handle); + +typedef struct { + int index; + int nAsync; + uv_async_t* asyncs; +} SAsyncPool; + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +void transDestroyAsyncPool(SAsyncPool* pool); +int transSendAsync(SAsyncPool* pool); + int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 069ebaeb8a..5037de1407 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -42,9 +42,10 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; - uv_loop_t* loop; - uv_async_t* cliAsync; // + pthread_t thread; + uv_loop_t* loop; + // uv_async_t* cliAsync; // + SAsyncPool* asyncPool; uv_timer_t* timer; void* pool; // conn pool queue msg; @@ -379,7 +380,7 @@ static void clientConnCb(uv_connect_t* req, int status) { static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("thread %p start to quit", pThrd); destroyCmsg(pMsg); - uv_close((uv_handle_t*)pThrd->cliAsync, NULL); + // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); uv_timer_stop(pThrd->timer); pThrd->quit = true; // uv__async_stop(pThrd->cliAsync); @@ -501,6 +502,7 @@ static void destroyCmsg(SCliMsg* pMsg) { destroyUserdata(&pMsg->msg); free(pMsg); } + static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); @@ -509,9 +511,7 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); pThrd->timer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->timer); @@ -529,7 +529,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { uv_stop(pThrd->loop); pthread_join(pThrd->thread, NULL); pthread_mutex_destroy(&pThrd->msgMtx); - free(pThrd->cliAsync); + transDestroyAsyncPool(pThrd->asyncPool); + // free(pThrd->cliAsync); free(pThrd->timer); free(pThrd->loop); free(pThrd); @@ -551,7 +552,8 @@ static void clientSendQuit(SCliThrdObj* thrd) { QUEUE_PUSH(&thrd->msg, &msg->q); pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + transSendAsync(thrd->asyncPool); + // uv_async_send(thrd->cliAsync); } void taosCloseClient(void* arg) { // impl later @@ -600,6 +602,10 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* QUEUE_PUSH(&thrd->msg, &cliMsg->q); pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + int start = taosGetTimestampUs(); + transSendAsync(thrd->asyncPool); + // uv_async_send(thrd->cliAsync); + int end = taosGetTimestampUs() - start; + // tError("client sent to rpc, time cost: %d", (int)end); } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ca39f85eb3..47eabd4320 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -245,4 +245,36 @@ int transDestroyBuffer(SConnBuffer* buf) { } transClearBuffer(buf); } + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { + static int sz = 20; + + SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + pool->index = 0; + pool->nAsync = sz; + pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + uv_async_init(loop, async, cb); + async->data = arg; + } + return pool; +} +void transDestroyAsyncPool(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + } + free(pool->asyncs); + free(pool); +} +int transSendAsync(SAsyncPool* pool) { + int idx = pool->index; + idx = idx % pool->nAsync; + // no need mutex here + if (pool->index++ > pool->nAsync) { + pool->index = 0; + } + return uv_async_send(&(pool->asyncs[idx])); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 475ef32b46..826b91dc02 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -22,7 +22,7 @@ typedef struct SSrvConn { uv_write_t* pWriter; uv_timer_t* pTimer; - uv_async_t* pWorkerAsync; + // uv_async_t* pWorkerAsync; queue queue; int ref; int persist; // persist connection or not @@ -50,11 +50,12 @@ typedef struct SSrvMsg { } SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - int fd; - uv_loop_t* loop; - uv_async_t* workerAsync; // + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + // uv_async_t* workerAsync; // queue msg; pthread_mutex_t msgMtx; void* pTransInst; @@ -469,7 +470,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - pConn->pWorkerAsync = pThrd->workerAsync; // thread safty + // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -512,10 +513,7 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); - pThrd->workerAsync->data = pThrd; - + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -665,7 +663,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } pthread_join(pThrd->thread, NULL); free(pThrd->loop); - free(pThrd->workerAsync); + transDestroyAsyncPool(pThrd->asyncPool); + + // free(pThrd->workerAsync); free(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { @@ -676,7 +676,8 @@ void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); - uv_async_send(pThrd->workerAsync); + transSendAsync(pThrd->asyncPool); + // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -716,8 +717,8 @@ void rpcSendResponse(const SRpcMsg* pMsg) { pthread_mutex_unlock(&pThrd->msgMtx); tDebug("conn %p start to send resp", pConn); - - uv_async_send(pThrd->workerAsync); + transSendAsync(pThrd->asyncPool); + // uv_async_send(pThrd->workerAsync); } #endif diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index f9ad20c065..37c189406a 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include #include "os.h" #include "rpcLog.h" @@ -52,6 +53,12 @@ static void *sendRequest(void *param) { tDebug("thread:%d, start to send request", pInfo->index); + tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + int u100 = 0; + int u500 = 0; + int u1000 = 0; + int u10000 = 0; + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); @@ -59,15 +66,28 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); + int64_t end = taosGetTimestampUs() - start; + if (end <= 100) { + u100++; + } else if (end > 100 && end <= 500) { + u500++; + } else if (end > 500 && end < 1000) { + u1000++; + } else { + u10000++; + } + tDebug("recv response succefully"); // usleep(100000000); } + tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); tDebug("thread:%d, it is over", pInfo->index); tcount++; @@ -163,8 +183,8 @@ int main(int argc, char *argv[]) { tInfo("client is initialized"); tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); - // gettimeofday(&systemTime, NULL); - // startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + gettimeofday(&systemTime, NULL); + startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); @@ -186,13 +206,12 @@ int main(int argc, char *argv[]) { usleep(1); } while (tcount < appThreads); - // gettimeofday(&systemTime, NULL); - // endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - // float usedTime = (endTime - startTime) / 1000.0f; // mseconds + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime) / 1000.0f; // mseconds - // tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - // tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, - // msgSize); + tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); int ch = getchar(); UNUSED(ch); From 17792e02706ec940e37d2a7962bb3a77074e5070 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 28 Jan 2022 13:46:01 +0800 Subject: [PATCH 24/42] fix show vgroups issue --- source/client/src/clientImpl.c | 8 ++++++-- source/client/src/clientMsgHandler.c | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 07fbf34793..5ab17ed7ae 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1136,8 +1136,12 @@ void* doFetchRow(SRequestObj* pRequest) { tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; - } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE && pResultInfo->pData != NULL) { - return NULL; + } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { + epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + if (pResultInfo->completed) { + return NULL; + } } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 6f74c29650..7e897287f6 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -201,6 +201,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; + pResInfo->completed = pRetrieve->completed; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); From 610c812ed76123bf7357aaa3d292107e969c415c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 14:32:21 +0800 Subject: [PATCH 25/42] mq support stable subscribe --- source/client/src/clientImpl.c | 65 +++++++++++++++++++-- source/client/test/clientTests.cpp | 6 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/libs/executor/src/executorimpl.c | 2 + source/libs/executor/test/executorTests.cpp | 2 +- source/libs/planner/src/physicalPlan.c | 3 +- src/connector/grafanaplugin | 1 + 8 files changed, 71 insertions(+), 13 deletions(-) create mode 160000 src/connector/grafanaplugin diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7e95d55ec7..fe65911780 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -624,14 +624,64 @@ _return: return pRequest; } -/*typedef SMqConsumeRsp tmq_message_t;*/ +static char *formatTimestamp(char *buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm *ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { if (code == -1) { - printf("discard\n"); + printf("msg discard\n"); return 0; } - SMqClientVg* pVg = (SMqClientVg*)param; + char pBuf[128]; + SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; + SMqClientVg* pVg = pParam->pVg; SMqConsumeRsp rsp; tDecodeSMqConsumeRsp(pMsg->pData, &rsp); if (rsp.numOfTopics == 0) { @@ -644,10 +694,11 @@ int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { /*printf("-----msg begin----\n");*/ printf("|"); for (int32_t i = 0; i < colNum; i++) { - printf(" %15s |", rsp.schemas->pSchema[i].name); + if (i == 0) printf(" %25s |", rsp.schemas->pSchema[i].name); + else printf(" %15s |", rsp.schemas->pSchema[i].name); } printf("\n"); - printf("=====================================\n"); + printf("===============================================\n"); int32_t sz = taosArrayGetSize(rsp.pBlockData); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); @@ -659,7 +710,8 @@ int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch(pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: - printf(" %15lu |", *(uint64_t*)var); + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); break; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: @@ -789,6 +841,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); pReq->offset = pVg->currentOffset+1; + *ppVg = pVg; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8cca1e6152..13f438f8f5 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -606,7 +606,7 @@ TEST(testCase, create_topic_stb_Test) { taos_free_result(pRes); - char* sql = "select * from st1"; + char* sql = "select ts, k from st1"; pRes = taos_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); taos_free_result(pRes); taos_close(pConn); @@ -657,18 +657,18 @@ TEST(testCase, tmq_subscribe_stb_Test) { tmq_subscribe(tmq, topic_list); while (1) { - tmq_message_t* msg = tmq_consume_poll(tmq, 1000); + tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); //printf("get msg\n"); //if (msg == NULL) break; } } -#endif TEST(testCase, tmq_consume_Test) { } TEST(testCase, tmq_commit_TEST) { } +#endif #if 0 TEST(testCase, projection_query_tables) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index daa3fa3722..d979ab06f6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -212,7 +212,7 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SA //pHandle->tbUid = pTableIdList; //} -static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle* pHandle, SArray* tbUidList) { +static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { return -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4e5246b0a8..ade623a736 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -824,8 +824,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pMeta }; pTopic->buffer.output[i].pReadHandle = pReadHandle; - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); } taosArrayPush(pConsumer->topics, pTopic); tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8e9a4ae861..2d3085c82b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5071,6 +5071,8 @@ static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { SStreamBlockScanInfo* pInfo = pOperator->info; SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + pBlockInfo->rows = 0; + while (tqNextDataBlock(pInfo->readerHandle)) { pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index b59be7d102..8381a7c585 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -219,7 +219,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - //int32_t code = qCreateExecTask((void*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); + int32_t code = qCreateExecTask((SReadHandle*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } #pragma GCC diagnostic pop diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 5d0600c820..20e8378c26 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -216,7 +216,8 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } else if (needSeqScan(pPlanNode)) { return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); } - return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan); + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; + return createUserTableScanNode(pPlanNode, pTable, type); } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin new file mode 160000 index 0000000000..4a4d79099b --- /dev/null +++ b/src/connector/grafanaplugin @@ -0,0 +1 @@ +Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d From 5bb010ef4af347cc72aa805d79297aff7a0b0046 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 15:24:09 +0800 Subject: [PATCH 26/42] refactor: move tmq out of clientImpl --- include/client/taos.h | 2 +- source/client/inc/clientInt.h | 15 + source/client/src/clientImpl.c | 676 ---------------------------- source/client/src/tmq.c | 691 +++++++++++++++++++++++++++++ source/client/test/CMakeLists.txt | 2 +- source/client/test/clientTests.cpp | 6 +- source/dnode/vnode/inc/vnode.h | 48 +- 7 files changed, 734 insertions(+), 706 deletions(-) create mode 100644 source/client/src/tmq.c diff --git a/include/client/taos.h b/include/client/taos.h index 6549821953..3db9046119 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -220,7 +220,7 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* c DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); -DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); +DLL_EXPORT tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 77249e3f0b..c977b4172c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -30,6 +30,16 @@ extern "C" { #include "tmsgtype.h" #include "trpc.h" #include "query.h" +#include "parser.h" + +#define CHECK_CODE_GOTO(expr, label) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + terrno = code; \ + goto label; \ + } \ + } while (0) #define HEARTBEAT_INTERVAL 1500 // ms @@ -219,6 +229,11 @@ void *doFetchRow(SRequestObj* pRequest); void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); + +int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); + +int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); + // --- heartbeat // global, called by mgmt int hbMgrInit(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fe65911780..8332f18512 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -12,102 +12,6 @@ #include "tpagedfile.h" #include "tref.h" -#define CHECK_CODE_GOTO(expr, label) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ - goto label; \ - } \ - } while (0) - -typedef struct SMqClientVg { - // statistics - int64_t pollCnt; - // offset - int64_t committedOffset; - int64_t currentOffset; - //connection info - int32_t vgId; - SEpSet epSet; -} SMqClientVg; - -typedef struct SMqClientTopic { - // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - int32_t nextVgIdx; - SArray* vgs; //SArray -} SMqClientTopic; - -struct tmq_resp_err_t { - int32_t code; -}; - -struct tmq_topic_vgroup_t { - char* topic; - int32_t vgId; - int64_t commitOffset; -}; - -struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; -}; - -typedef struct SMqConsumeCbParam { - tmq_t* tmq; - SMqClientVg* pVg; - tmq_message_t** retMsg; -} SMqConsumeCbParam; - -struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - char* ip; - uint16_t port; - tmq_commit_cb* commit_cb; -}; - -struct tmq_message_t { - SMqConsumeRsp rsp; -}; - - -tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); - return conf; -} - -int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); - } - if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); - } - return 0; -} - -struct tmq_t { - char groupId[256]; - char clientId[256]; - SRWLatch lock; - int64_t consumerId; - int64_t epoch; - int64_t status; - tsem_t rspSem; - STscObj* pTscObj; - tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - SArray* clientTopics; //SArray - //stat - int64_t pollCnt; -}; - static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); @@ -345,586 +249,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) return schedulerAsyncExecJob(pTransporter, pNodeList, pDag, pRequest->sqlstr, &pRequest->body.pQueryJob); } - - -tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = calloc(sizeof(tmq_t), 1); - if (pTmq == NULL) { - return NULL; - } - pTmq->pTscObj = (STscObj*)conn; - pTmq->status = 0; - pTmq->pollCnt = 0; - pTmq->epoch = 0; - taosInitRWLatch(&pTmq->lock); - strcpy(pTmq->clientId, conf->clientId); - strcpy(pTmq->groupId, conf->groupId); - pTmq->commit_cb = conf->commit_cb; - tsem_init(&pTmq->rspSem, 0, 0); - pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); - pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); - return pTmq; -} - -struct tmq_list_t { - int32_t cnt; - int32_t tot; - char* elems[]; -}; -tmq_list_t* tmq_list_new() { - tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); - if (ptr == NULL) { - return ptr; - } - ptr->cnt = 0; - ptr->tot = 8; - return ptr; -} - -int32_t tmq_list_append(tmq_list_t* ptr, char* src) { - if (ptr->cnt >= ptr->tot-1) return -1; - ptr->elems[ptr->cnt] = strdup(src); - ptr->cnt++; - return 0; -} - - -int32_t tmq_null_cb(void* param, const SDataBuf* pMsg, int32_t code) { - if (code == 0) { - // - } - // - return 0; -} - -TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { - SRequestObj *pRequest = NULL; - int32_t sz = topic_list->cnt; - //destroy ex - taosArrayDestroy(tmq->clientTopics); - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - - SCMSubscribeReq req; - req.topicNum = sz; - req.consumerId = tmq->consumerId; - req.consumerGroup = strdup(tmq->groupId); - req.topicNames = taosArrayInit(sz, sizeof(void*)); - - for (int i = 0; i < sz; i++) { - char* topicName = topic_list->elems[i]; - - SName name = {0}; - char* dbName = getDbOfConnection(tmq->pTscObj); - tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); - tNameFromString(&name, topicName, T_NAME_TABLE); - - char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); - if (topicFname == NULL) { - - } - tNameExtractFullName(&name, topicFname); - tscDebug("subscribe topic: %s", topicFname); - SMqClientTopic topic = { - .nextVgIdx = 0, - .sql = NULL, - .sqlLen = 0, - .topicId = 0, - .topicName = topicFname, - .vgs = NULL - }; - topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); - taosArrayPush(tmq->clientTopics, &topic); - /*SMqClientTopic topic = {*/ - /*.*/ - /*};*/ - taosArrayPush(req.topicNames, &topicFname); - } - - int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = malloc(tlen); - if(buf == NULL) { - goto _return; - } - - void* abuf = buf; - tSerializeSCMSubscribeReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); - if (pRequest == NULL) { - tscError("failed to malloc sqlObj"); - } - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - /*sendInfo->fp*/ - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - -_return: - /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ - /*}*/ - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} - -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { - conf->commit_cb = cb; -} - -SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { - tmq_t* pTmq = (void*)param; - SArray* pArray = taosArrayInit(0, sizeof(SKv)); - if (pArray == NULL) { - return NULL; - } - SKv kv = {0}; - kv.key = HEARTBEAT_KEY_MQ_TMP; - - SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); - if (pMqHb == NULL) { - return pArray; - } - pMqHb->consumerId = connKey.connId; - SArray* clientTopics = pTmq->clientTopics; - int sz = taosArrayGetSize(clientTopics); - for (int i = 0; i < sz; i++) { - SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); - /*if (pCTopic->vgId == -1) {*/ - /*pMqHb->status = 1;*/ - /*break;*/ - /*}*/ - } - kv.value = pMqHb; - kv.valueLen = sizeof(SMqHbMsg); - taosArrayPush(pArray, &kv); - - return pArray; -} - -tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); - if (pTmq == NULL) { - return NULL; - } - strcpy(pTmq->groupId, conf->groupId); - strcpy(pTmq->clientId, conf->clientId); - pTmq->pTscObj = (STscObj*)conn; - pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; - - return pTmq; -} - -TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { - STscObj *pTscObj = (STscObj*)taos; - SRequestObj *pRequest = NULL; - SQueryNode *pQueryNode = NULL; - char *pStr = NULL; - - terrno = TSDB_CODE_SUCCESS; - if (taos == NULL || topicName == NULL || sql == NULL) { - tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { - tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - goto _return; - } - - tscDebug("start to create topic, %s", topicName); - - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); - - SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo* ) pQueryNode; - pQueryStmtInfo->info.continueQuery = true; - - // todo check for invalid sql statement and return with error code - - SSchema *schema = NULL; - int32_t numOfCols = 0; - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); - - pStr = qDagToString(pRequest->body.pDag); - if(pStr == NULL) { - goto _return; - } - - printf("%s\n", pStr); - - // The topic should be related to a database that the queried table is belonged to. - SName name = {0}; - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName); - - tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB); - tNameFromString(&name, topicName, T_NAME_TABLE); - - char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; - tNameExtractFullName(&name, topicFname); - - SCMCreateTopicReq req = { - .name = (char*) topicFname, - .igExists = 1, - .physicalPlan = (char*) pStr, - .sql = (char*) sql, - .logicalPlan = "no logic plan", - }; - - int tlen = tSerializeSCMCreateTopicReq(NULL, &req); - void* buf = malloc(tlen); - if (buf == NULL) { - goto _return; - } - - void* abuf = buf; - tSerializeSCMCreateTopicReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - pRequest->type = TDMT_MND_CREATE_TOPIC; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - -_return: - qDestroyQuery(pQueryNode); - /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ - /*}*/ - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} - -static char *formatTimestamp(char *buf, int64_t val, int precision) { - time_t tt; - int32_t ms = 0; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - ms = val % 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (time_t)(val / 1000000); - ms = val % 1000000; - } else { - tt = (time_t)(val / 1000); - ms = val % 1000; - } - - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - if (tt <= 0 && ms < 0) { - tt--; - if (precision == TSDB_TIME_PRECISION_NANO) { - ms += 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - ms += 1000000; - } else { - ms += 1000; - } - } - - struct tm *ptm = localtime(&tt); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); - - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); - } else { - sprintf(buf + pos, ".%03d", ms); - } - - return buf; -} - -int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { - if (code == -1) { - printf("msg discard\n"); - return 0; - } - char pBuf[128]; - SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; - SMqClientVg* pVg = pParam->pVg; - SMqConsumeRsp rsp; - tDecodeSMqConsumeRsp(pMsg->pData, &rsp); - if (rsp.numOfTopics == 0) { - /*printf("no data\n");*/ - return 0; - } - int32_t colNum = rsp.schemas->nCols; - pVg->currentOffset = rsp.rspOffset; - /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ - /*printf("-----msg begin----\n");*/ - printf("|"); - for (int32_t i = 0; i < colNum; i++) { - if (i == 0) printf(" %25s |", rsp.schemas->pSchema[i].name); - else printf(" %15s |", rsp.schemas->pSchema[i].name); - } - printf("\n"); - printf("===============================================\n"); - int32_t sz = taosArrayGetSize(rsp.pBlockData); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); - int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < rows; j++) { - printf("|"); - for (int32_t k = 0; k < colNum; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch(pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); - break; - } - } - printf("\n"); - } - } - /*printf("\n-----msg end------\n");*/ - return 0; -} - -typedef struct SMqAskEpCbParam { - tmq_t* tmq; - int32_t wait; -} SMqAskEpCbParam; - -int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - tmq_t* tmq = pParam->tmq; - if (code != 0) { - printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); - if (pParam->wait) { - tsem_post(&tmq->rspSem); - } - return 0; - } - tscDebug("tmq ask ep cb called"); - bool set = false; - SMqCMGetSubEpRsp rsp; - tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); - int32_t sz = taosArrayGetSize(rsp.topics); - // TODO: lock - /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ - /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ - if (rsp.epoch != tmq->epoch) { - //TODO - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); - topic.topicName = strdup(pTopicEp->topic); - int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgSz; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - SMqClientVg clientVg = { - .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(tmq->clientTopics, &topic); - } - tmq->epoch = rsp.epoch; - } - if (set) { - atomic_store_64(&tmq->status, 1); - } - // unlock - /*tsem_post(&tmq->rspSem);*/ - if (pParam->wait) { - tsem_post(&tmq->rspSem); - } - free(pParam); - return 0; -} - -int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { - int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* buf = malloc(tlen); - if (buf == NULL) { - tscError("failed to malloc get subscribe ep buf"); - } - buf->consumerId = htobe64(tmq->consumerId); - strcpy(buf->cgroup, tmq->groupId); - - SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); - if (pRequest == NULL) { - tscError("failed to malloc subscribe ep request"); - } - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - - SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); - if (pParam == NULL) { - free(buf); - goto END; - } - pParam->tmq = tmq; - pParam->wait = wait; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmq_ask_ep_cb; - - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - -END: - if (wait) tsem_wait(&tmq->rspSem); - return 0; -} - -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, SMqClientVg** ppVg) { - SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); - if (pReq == NULL) { - return NULL; - } - pReq->reqType = type; - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; - strcpy(pReq->cgroup, tmq->groupId); - - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - strcpy(pReq->topic, pTopic->topicName); - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - pReq->offset = pVg->currentOffset+1; - *ppVg = pVg; - - pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); - return pReq; -} - -tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* tmq_message = NULL; - - int64_t status = atomic_load_64(&tmq->status); - tmqAsyncAskEp(tmq, status == 0 || taosArrayGetSize(tmq->clientTopics)); - - /*if (blocking_time < 0) blocking_time = 500;*/ - blocking_time = 1000; - - if (taosArrayGetSize(tmq->clientTopics) == 0) { - tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - usleep(blocking_time * 1000); - return NULL; - } - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); - if (taosArrayGetSize(pTopic->vgs) == 0) { - usleep(blocking_time * 1000); - return NULL; - } - - SMqClientVg* pVg = NULL; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, &pVg); - if (pReq == NULL) { - usleep(blocking_time * 1000); - return NULL; - } - - SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); - if (param == NULL) { - usleep(blocking_time * 1000); - return NULL; - } - param->tmq = tmq; - param->retMsg = &tmq_message; - param->pVg = pVg; - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = param; - sendInfo->fp = tmq_poll_cb_inner; - - /*printf("req offset: %ld\n", pReq->offset);*/ - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tmq->pollCnt++; - - usleep(blocking_time * 1000); - - return tmq_message; - - /*tsem_wait(&pRequest->body.rspSem);*/ - - /*if (body != NULL) {*/ - /*destroySendMsgInfo(body);*/ - /*}*/ - - /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ - /*pRequest->code = terrno;*/ - /*}*/ - - /*return pRequest;*/ -} - -tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - SMqConsumeReq req = {0}; - return NULL; -} - -void tmq_message_destroy(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; -} - - TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; if (sqlLen > (size_t) TSDB_MAX_ALLOWED_SQL_LEN) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c new file mode 100644 index 0000000000..632b1dddb4 --- /dev/null +++ b/source/client/src/tmq.c @@ -0,0 +1,691 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "clientInt.h" +#include "clientLog.h" +#include "parser.h" +#include "planner.h" +#include "scheduler.h" +#include "tdef.h" +#include "tep.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tnote.h" +#include "tpagedfile.h" +#include "tref.h" + +typedef struct SMqClientVg { + // statistics + int64_t pollCnt; + // offset + int64_t committedOffset; + int64_t currentOffset; + //connection info + int32_t vgId; + SEpSet epSet; +} SMqClientVg; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; //SArray +} SMqClientTopic; + +typedef struct SMqAskEpCbParam { + tmq_t* tmq; + int32_t wait; +} SMqAskEpCbParam; + +struct tmq_resp_err_t { + int32_t code; +}; + +struct tmq_topic_vgroup_t { + char* topic; + int32_t vgId; + int64_t commitOffset; +}; + +struct tmq_topic_vgroup_list_t { + int32_t cnt; + int32_t size; + tmq_topic_vgroup_t* elems; +}; + +typedef struct SMqConsumeCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + tmq_message_t** retMsg; +} SMqConsumeCbParam; + +struct tmq_conf_t { + char clientId[256]; + char groupId[256]; + char* ip; + uint16_t port; + tmq_commit_cb* commit_cb; +}; + +struct tmq_message_t { + SMqConsumeRsp rsp; +}; + + +tmq_conf_t* tmq_conf_new() { + tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + return conf; +} + +int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (strcmp(key, "group.id") == 0) { + strcpy(conf->groupId, value); + } + if (strcmp(key, "client.id") == 0) { + strcpy(conf->clientId, value); + } + return 0; +} + +struct tmq_t { + char groupId[256]; + char clientId[256]; + SRWLatch lock; + int64_t consumerId; + int64_t epoch; + int64_t status; + tsem_t rspSem; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + int32_t nextTopicIdx; + SArray* clientTopics; //SArray + //stat + int64_t pollCnt; +}; + +struct tmq_list_t { + int32_t cnt; + int32_t tot; + char* elems[]; +}; + +tmq_list_t* tmq_list_new() { + tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + if (ptr == NULL) { + return ptr; + } + ptr->cnt = 0; + ptr->tot = 8; + return ptr; +} + +int32_t tmq_list_append(tmq_list_t* ptr, char* src) { + if (ptr->cnt >= ptr->tot-1) return -1; + ptr->elems[ptr->cnt] = strdup(src); + ptr->cnt++; + return 0; +} + + +tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + if (pTmq == NULL) { + return NULL; + } + pTmq->pTscObj = (STscObj*)conn; + pTmq->status = 0; + pTmq->pollCnt = 0; + pTmq->epoch = 0; + taosInitRWLatch(&pTmq->lock); + strcpy(pTmq->clientId, conf->clientId); + strcpy(pTmq->groupId, conf->groupId); + pTmq->commit_cb = conf->commit_cb; + tsem_init(&pTmq->rspSem, 0, 0); + pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + return pTmq; +} + +TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { + SRequestObj *pRequest = NULL; + int32_t sz = topic_list->cnt; + //destroy ex + taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + + SCMSubscribeReq req; + req.topicNum = sz; + req.consumerId = tmq->consumerId; + req.consumerGroup = strdup(tmq->groupId); + req.topicNames = taosArrayInit(sz, sizeof(void*)); + + for (int i = 0; i < sz; i++) { + char* topicName = topic_list->elems[i]; + + SName name = {0}; + char* dbName = getDbOfConnection(tmq->pTscObj); + tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + if (topicFname == NULL) { + + } + tNameExtractFullName(&name, topicFname); + tscDebug("subscribe topic: %s", topicFname); + SMqClientTopic topic = { + .nextVgIdx = 0, + .sql = NULL, + .sqlLen = 0, + .topicId = 0, + .topicName = topicFname, + .vgs = NULL + }; + topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); + taosArrayPush(tmq->clientTopics, &topic); + /*SMqClientTopic topic = {*/ + /*.*/ + /*};*/ + taosArrayPush(req.topicNames, &topicFname); + } + + int tlen = tSerializeSCMSubscribeReq(NULL, &req); + void* buf = malloc(tlen); + if(buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMSubscribeReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + } + + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + /*sendInfo->fp*/ + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + +_return: + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { + conf->commit_cb = cb; +} + +SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { + tmq_t* pTmq = (void*)param; + SArray* pArray = taosArrayInit(0, sizeof(SKv)); + if (pArray == NULL) { + return NULL; + } + SKv kv = {0}; + kv.key = HEARTBEAT_KEY_MQ_TMP; + + SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); + if (pMqHb == NULL) { + return pArray; + } + pMqHb->consumerId = connKey.connId; + SArray* clientTopics = pTmq->clientTopics; + int sz = taosArrayGetSize(clientTopics); + for (int i = 0; i < sz; i++) { + SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); + /*if (pCTopic->vgId == -1) {*/ + /*pMqHb->status = 1;*/ + /*break;*/ + /*}*/ + } + kv.value = pMqHb; + kv.valueLen = sizeof(SMqHbMsg); + taosArrayPush(pArray, &kv); + + return pArray; +} + +tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { + tmq_t* pTmq = malloc(sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + strcpy(pTmq->groupId, conf->groupId); + strcpy(pTmq->clientId, conf->clientId); + pTmq->pTscObj = (STscObj*)conn; + pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; + + return pTmq; +} + +TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { + STscObj *pTscObj = (STscObj*)taos; + SRequestObj *pRequest = NULL; + SQueryNode *pQueryNode = NULL; + char *pStr = NULL; + + terrno = TSDB_CODE_SUCCESS; + if (taos == NULL || topicName == NULL || sql == NULL) { + tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { + tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + goto _return; + } + + tscDebug("start to create topic, %s", topicName); + + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); + + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo* ) pQueryNode; + pQueryStmtInfo->info.continueQuery = true; + + // todo check for invalid sql statement and return with error code + + SSchema *schema = NULL; + int32_t numOfCols = 0; + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); + + pStr = qDagToString(pRequest->body.pDag); + if(pStr == NULL) { + goto _return; + } + + printf("%s\n", pStr); + + // The topic should be related to a database that the queried table is belonged to. + SName name = {0}; + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName); + + tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; + tNameExtractFullName(&name, topicFname); + + SCMCreateTopicReq req = { + .name = (char*) topicFname, + .igExists = 1, + .physicalPlan = (char*) pStr, + .sql = (char*) sql, + .logicalPlan = "no logic plan", + }; + + int tlen = tSerializeSCMCreateTopicReq(NULL, &req); + void* buf = malloc(tlen); + if (buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMCreateTopicReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + pRequest->type = TDMT_MND_CREATE_TOPIC; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + +_return: + qDestroyQuery(pQueryNode); + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + +static char *formatTimestamp(char *buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm *ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} + +int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { + if (code == -1) { + printf("msg discard\n"); + return 0; + } + char pBuf[128]; + SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; + SMqClientVg* pVg = pParam->pVg; + SMqConsumeRsp rsp; + tDecodeSMqConsumeRsp(pMsg->pData, &rsp); + if (rsp.numOfTopics == 0) { + /*printf("no data\n");*/ + return 0; + } + int32_t colNum = rsp.schemas->nCols; + pVg->currentOffset = rsp.rspOffset; + /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ + /*printf("-----msg begin----\n");*/ + printf("|"); + for (int32_t i = 0; i < colNum; i++) { + if (i == 0) printf(" %25s |", rsp.schemas->pSchema[i].name); + else printf(" %15s |", rsp.schemas->pSchema[i].name); + } + printf("\n"); + printf("===============================================\n"); + int32_t sz = taosArrayGetSize(rsp.pBlockData); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch(pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t*)var); + break; + } + } + printf("\n"); + } + } + /*printf("\n-----msg end------\n");*/ + return 0; +} + +int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + tmq_t* tmq = pParam->tmq; + if (code != 0) { + printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + return 0; + } + tscDebug("tmq ask ep cb called"); + bool set = false; + SMqCMGetSubEpRsp rsp; + tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); + int32_t sz = taosArrayGetSize(rsp.topics); + // TODO: lock + /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ + if (rsp.epoch != tmq->epoch) { + //TODO + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); + topic.topicName = strdup(pTopicEp->topic); + int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgSz; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + SMqClientVg clientVg = { + .pollCnt = 0, + .committedOffset = -1, + .currentOffset = -1, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet + }; + taosArrayPush(topic.vgs, &clientVg); + set = true; + } + taosArrayPush(tmq->clientTopics, &topic); + } + tmq->epoch = rsp.epoch; + } + if (set) { + atomic_store_64(&tmq->status, 1); + } + // unlock + /*tsem_post(&tmq->rspSem);*/ + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + free(pParam); + return 0; +} + +int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { + int32_t tlen = sizeof(SMqCMGetSubEpReq); + SMqCMGetSubEpReq* buf = malloc(tlen); + if (buf == NULL) { + tscError("failed to malloc get subscribe ep buf"); + } + buf->consumerId = htobe64(tmq->consumerId); + strcpy(buf->cgroup, tmq->groupId); + + SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); + if (pRequest == NULL) { + tscError("failed to malloc subscribe ep request"); + } + + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + + SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + free(buf); + goto END; + } + pParam->tmq = tmq; + pParam->wait = wait; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmq_ask_ep_cb; + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + +END: + if (wait) tsem_wait(&tmq->rspSem); + return 0; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, SMqClientVg** ppVg) { + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + if (pReq == NULL) { + return NULL; + } + pReq->reqType = type; + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + strcpy(pReq->cgroup, tmq->groupId); + + tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); + strcpy(pReq->topic, pTopic->topicName); + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + pReq->offset = pVg->currentOffset+1; + *ppVg = pVg; + + pReq->head.vgId = htonl(pVg->vgId); + pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + return pReq; +} + +tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* tmq_message = NULL; + + int64_t status = atomic_load_64(&tmq->status); + tmqAsyncAskEp(tmq, status == 0 || taosArrayGetSize(tmq->clientTopics)); + + /*if (blocking_time < 0) blocking_time = 500;*/ + blocking_time = 1000; + + if (taosArrayGetSize(tmq->clientTopics) == 0) { + tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); + usleep(blocking_time * 1000); + return NULL; + } + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); + if (taosArrayGetSize(pTopic->vgs) == 0) { + usleep(blocking_time * 1000); + return NULL; + } + + SMqClientVg* pVg = NULL; + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, &pVg); + if (pReq == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + + SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + if (param == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + param->tmq = tmq; + param->retMsg = &tmq_message; + param->pVg = pVg; + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) }; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmq_poll_cb_inner; + + /*printf("req offset: %ld\n", pReq->offset);*/ + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + tmq->pollCnt++; + + usleep(blocking_time * 1000); + + return tmq_message; + + /*tsem_wait(&pRequest->body.rspSem);*/ + + /*if (body != NULL) {*/ + /*destroySendMsgInfo(body);*/ + /*}*/ + + /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ + /*pRequest->code = terrno;*/ + /*}*/ + + /*return pRequest;*/ +} + +tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { + SMqConsumeReq req = {0}; + return NULL; +} + +void tmq_message_destroy(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; +} + + +static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->msgInfo.pData); + tfree(pMsgBody); +} diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 6886206363..3614e4364b 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 13f438f8f5..c42b023d12 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -625,7 +625,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg1"); - tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_list_t* topic_list = tmq_list_new(); tmq_list_append(topic_list, "test_ctb_topic_1"); @@ -637,6 +637,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) { //if (msg == NULL) break; } } +#endif TEST(testCase, tmq_subscribe_stb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -650,7 +651,7 @@ TEST(testCase, tmq_subscribe_stb_Test) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); - tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_list_t* topic_list = tmq_list_new(); tmq_list_append(topic_list, "test_stb_topic_1"); @@ -668,7 +669,6 @@ TEST(testCase, tmq_consume_Test) { TEST(testCase, tmq_commit_TEST) { } -#endif #if 0 TEST(testCase, projection_query_tables) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d979ab06f6..7549772613 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -40,7 +40,6 @@ typedef struct STqCfg { int32_t reserved; } STqCfg; - typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; @@ -69,18 +68,18 @@ typedef struct { } SVnodeOpt; typedef struct STqReadHandle { - int64_t ver; - uint64_t tbUid; - SHashObj* tbIdHash; - const SSubmitMsg* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - SMeta* pVnodeMeta; - SArray* pColIdList; //SArray - int32_t sver; - SSchemaWrapper* pSchemaWrapper; - STSchema* pSchema; + int64_t ver; + uint64_t tbUid; + SHashObj *tbIdHash; + const SSubmitMsg *pMsg; + SSubmitBlk *pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; + SMeta *pVnodeMeta; + SArray *pColIdList; // SArray + int32_t sver; + SSchemaWrapper *pSchemaWrapper; + STSchema *pSchema; } STqReadHandle; /* ------------------------ SVnode ------------------------ */ @@ -202,35 +201,34 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); /* ------------------------- TQ QUERY -------------------------- */ -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta); +STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); -static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { +static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) { pReadHandle->pColIdList = pColIdList; } -//static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { - //pHandle->tbUid = pTableIdList; +// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { +// pHandle->tbUid = pTableIdList; //} -static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { return -1; } for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); + int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); - //pHandle->tbUid = tbUid; + // pHandle->tbUid = tbUid; } return 0; } -void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle* pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); +void tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitMsg *pMsg, int64_t ver); +bool tqNextDataBlock(STqReadHandle *pHandle); +int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); // return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); - +SArray *tqRetrieveDataBlock(STqReadHandle *pHandle); #ifdef __cplusplus } From a703b7024d830246ac11a10674afddddf2bbe6e1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 15:57:38 +0800 Subject: [PATCH 27/42] support select * from stable --- source/dnode/vnode/src/tq/tq.c | 6 +++++- source/util/src/tlog.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ade623a736..f3163fb515 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -916,6 +916,11 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t numOfCols = pHandle->pSchema->numOfCols; int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + //TODO: stable case + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); if (pArray == NULL) { return NULL; @@ -928,7 +933,6 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { j++; } SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; - ASSERT(pColSchema->colId == colId); SColumnInfoData colInfo = {0}; int sz = numOfRows * pColSchema->bytes; colInfo.info.bytes = pColSchema->bytes; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c7b1350591..5ce69f1c2e 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -92,7 +92,7 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; -int32_t tqDebugFlag = 131; +int32_t tqDebugFlag = 135; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; From ff57f57c57ea0191a9da4cb0aeaaea3c68bbe13a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 17:19:12 +0800 Subject: [PATCH 28/42] refactor tmq --- include/client/taos.h | 63 ++++++++++++------ source/client/src/tmq.c | 144 ++++++++++++++++++++++------------------ 2 files changed, 121 insertions(+), 86 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 3db9046119..029aad8715 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -92,17 +92,6 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); -typedef struct tmq_t tmq_t; -typedef struct tmq_conf_t tmq_conf_t; -typedef struct tmq_list_t tmq_list_t; - -typedef struct tmq_message_t tmq_message_t; -typedef struct tmq_message_topic_t tmq_message_topic_t; -typedef struct tmq_message_tb_t tmq_message_tb_t; -typedef struct tmq_tb_iter_t tmq_tb_iter_t; -typedef struct tmq_message_col_t tmq_message_col_t; -typedef struct tmq_col_iter_t tmq_col_iter_t; - typedef struct TAOS_BIND { int buffer_type; void * buffer; @@ -205,27 +194,59 @@ DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ -typedef struct tmq_resp_err_t tmq_resp_err_t; + +enum tmq_resp_err_t { + TMQ_RESP_ERR__SUCCESS = 0, + TMQ_RESP_ERR__FAIL = 1, +}; + +typedef enum tmq_resp_err_t tmq_resp_err_t; + +typedef struct tmq_t tmq_t; typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; +typedef struct tmq_message_t tmq_message_t; + typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); DLL_EXPORT tmq_list_t* tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*); -DLL_EXPORT tmq_conf_t* tmq_conf_new(); - -DLL_EXPORT int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); -DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb); - DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); - DLL_EXPORT tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); -DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); - +/* ------------------------TMQ CONSUMER INTERFACE------------------------ */ +DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +#endif DLL_EXPORT tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time); -DLL_EXPORT tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t* tmq); +DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); +DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups); +#endif +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); +#endif +/* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ + +enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, +}; + +typedef enum tmq_conf_res_t tmq_conf_res_t; + +DLL_EXPORT tmq_conf_t* tmq_conf_new(); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t* conf); +DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb); #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 632b1dddb4..a4af40285e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -26,6 +26,51 @@ #include "tpagedfile.h" #include "tref.h" +struct tmq_list_t { + int32_t cnt; + int32_t tot; + char* elems[]; +}; +struct tmq_topic_vgroup_t { + char* topic; + int32_t vgId; + int64_t commitOffset; +}; + +struct tmq_topic_vgroup_list_t { + int32_t cnt; + int32_t size; + tmq_topic_vgroup_t* elems; +}; + +struct tmq_conf_t { + char clientId[256]; + char groupId[256]; + /*char* ip;*/ + /*uint16_t port;*/ + tmq_commit_cb* commit_cb; +}; + +struct tmq_t { + char groupId[256]; + char clientId[256]; + SRWLatch lock; + int64_t consumerId; + int64_t epoch; + int64_t status; + tsem_t rspSem; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + int32_t nextTopicIdx; + SArray* clientTopics; //SArray + //stat + int64_t pollCnt; +}; + +struct tmq_message_t { + SMqConsumeRsp rsp; +}; + typedef struct SMqClientVg { // statistics int64_t pollCnt; @@ -47,83 +92,43 @@ typedef struct SMqClientTopic { SArray* vgs; //SArray } SMqClientTopic; + typedef struct SMqAskEpCbParam { tmq_t* tmq; int32_t wait; } SMqAskEpCbParam; -struct tmq_resp_err_t { - int32_t code; -}; - -struct tmq_topic_vgroup_t { - char* topic; - int32_t vgId; - int64_t commitOffset; -}; - -struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; -}; - typedef struct SMqConsumeCbParam { tmq_t* tmq; SMqClientVg* pVg; tmq_message_t** retMsg; } SMqConsumeCbParam; -struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - char* ip; - uint16_t port; - tmq_commit_cb* commit_cb; -}; - -struct tmq_message_t { - SMqConsumeRsp rsp; -}; - +typedef struct SMqSubscribeCbParam { + tmq_t* tmq; + tsem_t rspSem; + tmq_resp_err_t rspErr; +} SMqSubscribeCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); return conf; } -int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { +void tmq_conf_destroy(tmq_conf_t* conf) { + if(conf) free(conf); +} + +tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { if (strcmp(key, "group.id") == 0) { strcpy(conf->groupId, value); } if (strcmp(key, "client.id") == 0) { strcpy(conf->clientId, value); } - return 0; + return TMQ_CONF_OK; } -struct tmq_t { - char groupId[256]; - char clientId[256]; - SRWLatch lock; - int64_t consumerId; - int64_t epoch; - int64_t status; - tsem_t rspSem; - STscObj* pTscObj; - tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - SArray* clientTopics; //SArray - //stat - int64_t pollCnt; -}; - -struct tmq_list_t { - int32_t cnt; - int32_t tot; - char* elems[]; -}; - tmq_list_t* tmq_list_new() { tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); if (ptr == NULL) { @@ -141,6 +146,12 @@ int32_t tmq_list_append(tmq_list_t* ptr, char* src) { return 0; } +int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; + pParam->rspErr = code; + tsem_post(&pParam->rspSem); + return 0; +} tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = calloc(sizeof(tmq_t), 1); @@ -161,7 +172,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } -TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { +tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SRequestObj *pRequest = NULL; int32_t sz = topic_list->cnt; //destroy ex @@ -219,27 +230,31 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tscError("failed to malloc sqlObj"); } + SMqSubscribeCbParam param = { + .rspErr = TMQ_RESP_ERR__SUCCESS, + .tmq = tmq + }; + tsem_init(¶m.rspSem, 0, 0); + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - /*sendInfo->fp*/ + sendInfo->param = ¶m; + sendInfo->fp = tmqSubscribeCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(&pRequest->body.rspSem); + tsem_wait(¶m.rspSem); + tsem_destroy(¶m.rspSem); _return: /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ /*}*/ - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; + return param.rspErr; } void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { @@ -611,10 +626,10 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_message = NULL; int64_t status = atomic_load_64(&tmq->status); - tmqAsyncAskEp(tmq, status == 0 || taosArrayGetSize(tmq->clientTopics)); + tmqAsyncAskEp(tmq, taosArrayGetSize(tmq->clientTopics)); /*if (blocking_time < 0) blocking_time = 500;*/ - blocking_time = 1000; + blocking_time = 1; if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); @@ -674,16 +689,15 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } -tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { SMqConsumeReq req = {0}; - return NULL; + return 0; } void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; } - static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); tfree(pMsgBody->msgInfo.pData); From 2550d42cd3cf7d870efe46bbfbcc397d26b78ef5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 17:56:12 +0800 Subject: [PATCH 29/42] fix mem leak --- include/common/common.h | 18 ++++++++++++++++++ source/client/test/clientTests.cpp | 2 +- source/dnode/vnode/src/tq/tq.c | 7 +++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/include/common/common.h b/include/common/common.h index bbde65b48f..092a666e73 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -174,6 +174,24 @@ static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { return buf; } +static FORCE_INLINE void destroySSDataBlock(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return; + } + + //int32_t numOfOutput = pBlock->info.numOfCols; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + for(int32_t i = 0; i < sz; ++i) { + SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + tfree(pBlock); +} + + //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index c42b023d12..7ef06397f0 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -606,7 +606,7 @@ TEST(testCase, create_topic_stb_Test) { taos_free_result(pRes); - char* sql = "select ts, k from st1"; + char* sql = "select * from st1"; pRes = taos_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); taos_free_result(pRes); taos_close(pConn); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f3163fb515..49d15e6148 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -785,6 +785,13 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } void* abuf = buf; tEncodeSMqConsumeRsp(&abuf, &rsp); + if (rsp.pBlockData) { + for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) { + SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i); + destroySSDataBlock(pBlock); + } + free(rsp.pBlockData); + } pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; From 688a85c9c3076bee5ab148aad6e31592fd944afd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 18:14:38 +0800 Subject: [PATCH 30/42] fix mem leak --- source/dnode/mnode/impl/src/mndSubscribe.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index fa8c78f4b1..62083a3d03 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -122,7 +122,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { if (changed || found) { SSdbRaw *pRaw = mndSubActionEncode(pSub); sdbSetRawStatus(pRaw, SDB_STATUS_READY); - sdbWriteNotFree(pMnode->pSdb, pRaw); + sdbWrite(pMnode->pSdb, pRaw); } mndReleaseSubscribe(pMnode, pSub); } @@ -434,6 +434,7 @@ SUB_DECODE_OVER: if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); // TODO free subscribeobj + tfree(buf); tfree(pRow); return NULL; } From 108fd4cfc36926fc25cda1eb39d714c9af679b5f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 18:39:34 +0800 Subject: [PATCH 31/42] fix mem leak --- source/client/src/tmq.c | 4 +++- source/dnode/mnode/impl/inc/mndDef.h | 14 ++++++++++++++ source/dnode/mnode/impl/src/mndSubscribe.c | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index a4af40285e..6603c4019f 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -565,6 +565,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { + goto END; tscError("failed to malloc get subscribe ep buf"); } buf->consumerId = htobe64(tmq->consumerId); @@ -572,6 +573,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); if (pRequest == NULL) { + goto END; tscError("failed to malloc subscribe ep request"); } @@ -579,7 +581,6 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { - free(buf); goto END; } pParam->tmq = tmq; @@ -596,6 +597,7 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); END: + tfree(buf); if (wait) tsem_wait(&tmq->rspSem); return 0; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9cce25ed6a..1b7383bee1 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -393,6 +393,12 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu return buf; } +static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { + if (pConsumerEp) { + tfree(pConsumerEp->qmsg); + } +} + // unit for rebalance typedef struct SMqSubscribeObj { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -571,6 +577,14 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return buf; } +static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { + if (pSub->availConsumer) taosArrayDestroy(pSub->availConsumer); + if (pSub->assigned) taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); + if (pSub->unassignedVg) taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); + if (pSub->idleConsumer) taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + if (pSub->lostConsumer) taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); +} + typedef struct SMqCGroup { char name[TSDB_CONSUMER_GROUP_LEN]; int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 62083a3d03..0da5b5f27e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -449,6 +449,7 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { mTrace("subscribe:%s, perform delete action", pSub->key); + tDeleteSMqSubscribeObj(pSub); return 0; } From 19892cfaf78612e3c8d78899764d616ab0d346a4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 18:58:22 +0800 Subject: [PATCH 32/42] fix double free --- source/client/src/tmq.c | 1 - source/dnode/mnode/impl/inc/mndDef.h | 33 ++++++++++++++++------ source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6603c4019f..f9ade7287f 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -597,7 +597,6 @@ int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); END: - tfree(buf); if (wait) tsem_wait(&tmq->rspSem); return 0; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1b7383bee1..a9a633cf63 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -526,7 +526,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->assigned, &cEp); } @@ -539,7 +539,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->lostConsumer, &cEp); } @@ -553,7 +553,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->idleConsumer, &cEp); } @@ -569,7 +569,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->unassignedVg, &cEp); } @@ -578,11 +578,26 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) } static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { - if (pSub->availConsumer) taosArrayDestroy(pSub->availConsumer); - if (pSub->assigned) taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); - if (pSub->unassignedVg) taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); - if (pSub->idleConsumer) taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); - if (pSub->lostConsumer) taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + if (pSub->availConsumer) { + taosArrayDestroy(pSub->availConsumer); + pSub->availConsumer = NULL; + } + if (pSub->assigned) { + taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); + pSub->assigned = NULL; + } + if (pSub->unassignedVg) { + taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); + pSub->unassignedVg = NULL; + } + if (pSub->idleConsumer) { + taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + pSub->idleConsumer = NULL; + } + if (pSub->lostConsumer) { + taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + pSub->lostConsumer = NULL; + } } typedef struct SMqCGroup { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 0da5b5f27e..47d7e14e45 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -292,7 +292,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas return -1; } - SMqConsumerEp CEp; + SMqConsumerEp CEp = {0}; CEp.status = 0; CEp.consumerId = -1; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; From 8f24333d7d3594401ebb245f620194f4adb3b082 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 19:10:18 +0800 Subject: [PATCH 33/42] fix mem leak --- include/common/tmsg.h | 8 ++++++++ source/client/src/tmq.c | 1 + source/dnode/mnode/impl/src/mndConsumer.c | 2 ++ source/dnode/mnode/impl/src/mndSubscribe.c | 3 ++- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 154973e20f..a5b9d356f8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1744,6 +1744,10 @@ typedef struct SMqCMGetSubEpRsp { SArray* topics; // SArray } SMqCMGetSubEpRsp; +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + taosArrayDestroy(pSubTopicEp->vgs); +} + static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); @@ -1757,6 +1761,10 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { return buf; } +static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) { + taosArrayDestroyEx(pRsp->topics, (void (*)(void*)) tDeleteSMqSubTopicEp); +} + static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { int32_t tlen = 0; tlen += taosEncodeString(buf, pTopicEp->topic); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index f9ade7287f..ccdd1e64cb 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -557,6 +557,7 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->wait) { tsem_post(&tmq->rspSem); } + tDeleteSMqCMGetSubEpRsp(&rsp); free(pParam); return 0; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index e642b578fa..6fd61baee3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -75,6 +75,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_SUCCESS; CM_ENCODE_OVER: + tfree(buf); if (terrno != 0) { mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -117,6 +118,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; CM_DECODE_OVER: + tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 47d7e14e45..8e62d48e61 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -388,6 +388,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_SUCCESS; SUB_ENCODE_OVER: + tfree(buf); if (terrno != 0) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -431,10 +432,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; SUB_DECODE_OVER: + tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); // TODO free subscribeobj - tfree(buf); tfree(pRow); return NULL; } From ee783080f341580cff84005cd90c60b28f474cc5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 19:22:48 +0800 Subject: [PATCH 34/42] fix mem leak --- include/common/common.h | 15 ++++++++++++++- source/client/src/tmq.c | 4 ++++ source/dnode/vnode/src/tq/tq.c | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index 092a666e73..16a691c338 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -174,7 +174,7 @@ static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { return buf; } -static FORCE_INLINE void destroySSDataBlock(SSDataBlock* pBlock) { +static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { if (pBlock == NULL) { return; } @@ -192,6 +192,19 @@ static FORCE_INLINE void destroySSDataBlock(SSDataBlock* pBlock) { } +static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { + if (pRsp->schemas) { + if (pRsp->schemas->nCols) { + tfree(pRsp->schemas->pSchema); + } + free(pRsp->schemas); + } + for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); + tDeleteSSDataBlock(pDataBlock); + } +} + //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ccdd1e64cb..26177f466e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -456,6 +456,7 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { if (code == -1) { printf("msg discard\n"); + free(param); return 0; } char pBuf[128]; @@ -465,6 +466,7 @@ int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqConsumeRsp(pMsg->pData, &rsp); if (rsp.numOfTopics == 0) { /*printf("no data\n");*/ + free(param); return 0; } int32_t colNum = rsp.schemas->nCols; @@ -501,6 +503,8 @@ int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { printf("\n"); } } + tDeleteSMqConsumeRsp(&rsp); + free(param); /*printf("\n-----msg end------\n");*/ return 0; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 49d15e6148..7299710586 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -788,7 +788,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (rsp.pBlockData) { for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) { SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i); - destroySSDataBlock(pBlock); + tDeleteSSDataBlock(pBlock); } free(rsp.pBlockData); } From 5aa0f80c913d78a1ea6a538c5097e14f5a2c4ef0 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 19:29:18 +0800 Subject: [PATCH 35/42] fix mem leak --- include/common/common.h | 12 +++++++----- source/dnode/vnode/src/tq/tq.c | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index 16a691c338..bad9adedf5 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -188,7 +188,7 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); tfree(pBlock->pBlockAgg); - tfree(pBlock); + //tfree(pBlock); } @@ -199,10 +199,12 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { } free(pRsp->schemas); } - for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { - SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); - tDeleteSSDataBlock(pDataBlock); - } + taosArrayDestroyEx(pRsp->pBlockData, (void(*)(void*))tDeleteSSDataBlock); + pRsp->pBlockData = NULL; + //for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + //SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); + //tDeleteSSDataBlock(pDataBlock); + //} } //====================================================================================================================== diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7299710586..139120a46d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -786,11 +786,13 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { void* abuf = buf; tEncodeSMqConsumeRsp(&abuf, &rsp); if (rsp.pBlockData) { - for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) { - SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i); - tDeleteSSDataBlock(pBlock); - } - free(rsp.pBlockData); + taosArrayDestroyEx(rsp.pBlockData, (void(*)(void*))tDeleteSSDataBlock); + rsp.pBlockData = NULL; + /*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/ + /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ + /*tDeleteSSDataBlock(pBlock);*/ + /*}*/ + /*taosArrayDestroy(rsp.pBlockData);*/ } pMsg->pCont = buf; pMsg->contLen = tlen; From a1d143384ab963ec2fa61ef8781b1742a318a8c1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 28 Jan 2022 19:58:56 +0800 Subject: [PATCH 36/42] fix mem leak --- include/common/common.h | 4 ++-- source/client/src/tmq.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- source/dnode/mnode/impl/src/mndTopic.c | 7 +++---- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index bad9adedf5..31f905d47f 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -119,7 +119,7 @@ static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { buf = taosDecodeFixedI32(buf, &sz); pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); for (int32_t i = 0; i < sz; i++) { - SColumnInfoData data; + SColumnInfoData data = {0}; buf = taosDecodeFixedI16(buf, &data.info.colId); buf = taosDecodeFixedI16(buf, &data.info.type); buf = taosDecodeFixedI16(buf, &data.info.bytes); @@ -167,7 +167,7 @@ static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { buf = taosDecodeFixedI32(buf, &sz); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { - SSDataBlock block; + SSDataBlock block = {0}; tDecodeDataBlock(buf, &block); taosArrayPush(pRsp->pBlockData, &block); } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 26177f466e..421c8fae30 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -462,7 +462,7 @@ int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { char pBuf[128]; SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; SMqClientVg* pVg = pParam->pVg; - SMqConsumeRsp rsp; + SMqConsumeRsp rsp = {0}; tDecodeSMqConsumeRsp(pMsg->pData, &rsp); if (rsp.numOfTopics == 0) { /*printf("no data\n");*/ diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8e62d48e61..47bdefea3d 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -69,7 +69,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; - SMqCMGetSubEpRsp rsp; + SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); int64_t currentTs = taosGetTimestampMs(); @@ -134,7 +134,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } void *abuf = buf; tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); - // TODO: free rsp + tDeleteSMqCMGetSubEpRsp(&rsp); pMsg->pCont = buf; pMsg->contLen = tlen; return 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 6b4cb4ba59..33def69a68 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -240,15 +240,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); topicObj.dbUid = pDb->uid; topicObj.version = 1; - topicObj.sql = strdup(pCreate->sql); - topicObj.physicalPlan = strdup(pCreate->physicalPlan); - topicObj.logicalPlan = strdup(pCreate->logicalPlan); + topicObj.sql = pCreate->sql; + topicObj.physicalPlan = pCreate->physicalPlan; + topicObj.logicalPlan = pCreate->logicalPlan; topicObj.sqlLen = strlen(pCreate->sql); SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); if (pTopicRaw == NULL) return -1; if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; - // TODO: replace with trans to support recovery return sdbWrite(pMnode->pSdb, pTopicRaw); } From 35348528125c2aaa4ebf6fbe2c4dcb88baae6ff2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 29 Jan 2022 04:45:01 +0800 Subject: [PATCH 37/42] improve mq consume --- include/client/taos.h | 39 +- include/common/tmsg.h | 7 +- source/client/src/tmq.c | 431 +++++++++------- source/client/test/clientTests.cpp | 18 +- source/dnode/mnode/impl/inc/mndDef.h | 3 +- source/dnode/mnode/impl/src/mndSubscribe.c | 28 +- source/dnode/vnode/src/inc/tqMetaStore.h | 1 + source/dnode/vnode/src/tq/tq.c | 566 ++------------------- source/dnode/vnode/src/tq/tqMetaStore.c | 28 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 2 + source/libs/wal/src/walRead.c | 9 +- 11 files changed, 377 insertions(+), 755 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 029aad8715..ac2893651d 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -202,34 +202,36 @@ enum tmq_resp_err_t { typedef enum tmq_resp_err_t tmq_resp_err_t; -typedef struct tmq_t tmq_t; -typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; +typedef struct tmq_t tmq_t; +typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; -typedef struct tmq_conf_t tmq_conf_t; -typedef struct tmq_list_t tmq_list_t; -typedef struct tmq_message_t tmq_message_t; +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; +typedef struct tmq_message_t tmq_message_t; -typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); -DLL_EXPORT tmq_list_t* tmq_list_new(); -DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); -DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); -DLL_EXPORT tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); +DLL_EXPORT tmq_list_t *tmq_list_new(); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, char *); + +DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); +DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT void tmq_message_destroy(tmq_message_t* tmq_message); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ -DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); +DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); #if 0 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); #endif -DLL_EXPORT tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time); +DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); #if 0 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t* tmq); DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups); #endif -DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async); +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); #endif @@ -243,10 +245,13 @@ enum tmq_conf_res_t { typedef enum tmq_conf_res_t tmq_conf_res_t; -DLL_EXPORT tmq_conf_t* tmq_conf_new(); -DLL_EXPORT void tmq_conf_destroy(tmq_conf_t* conf); -DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); -DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb); +DLL_EXPORT tmq_conf_t *tmq_conf_new(); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); +DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); + +//temporary used function for demo only +void tmqShowMsg(tmq_message_t* tmq_message); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5b9d356f8..7cd8ffb6f3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1570,6 +1570,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { } typedef struct SMqSetCVgReq { + int64_t leftForVer; int32_t vgId; int64_t oldConsumerId; int64_t newConsumerId; @@ -1604,6 +1605,7 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->leftForVer); tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); @@ -1612,12 +1614,13 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeString(buf, (char*)pReq->qmsg); + tlen += taosEncodeString(buf, pReq->qmsg); //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->leftForVer); buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); @@ -1626,7 +1629,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = taosDecodeString(buf, (char**)&pReq->qmsg); + buf = taosDecodeString(buf, &pReq->qmsg); //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 421c8fae30..d529662008 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -34,18 +34,18 @@ struct tmq_list_t { struct tmq_topic_vgroup_t { char* topic; int32_t vgId; - int64_t commitOffset; + int64_t offset; }; struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; + int32_t cnt; + int32_t size; tmq_topic_vgroup_t* elems; }; struct tmq_conf_t { - char clientId[256]; - char groupId[256]; + char clientId[256]; + char groupId[256]; /*char* ip;*/ /*uint16_t port;*/ tmq_commit_cb* commit_cb; @@ -62,9 +62,9 @@ struct tmq_t { STscObj* pTscObj; tmq_commit_cb* commit_cb; int32_t nextTopicIdx; - SArray* clientTopics; //SArray - //stat - int64_t pollCnt; + SArray* clientTopics; // SArray + // stat + int64_t pollCnt; }; struct tmq_message_t { @@ -77,7 +77,7 @@ typedef struct SMqClientVg { // offset int64_t committedOffset; int64_t currentOffset; - //connection info + // connection info int32_t vgId; SEpSet epSet; } SMqClientVg; @@ -89,9 +89,14 @@ typedef struct SMqClientTopic { char* topicName; int64_t topicId; int32_t nextVgIdx; - SArray* vgs; //SArray + SArray* vgs; // SArray } SMqClientTopic; +typedef struct SMqSubscribeCbParam { + tmq_t* tmq; + tsem_t rspSem; + tmq_resp_err_t rspErr; +} SMqSubscribeCbParam; typedef struct SMqAskEpCbParam { tmq_t* tmq; @@ -102,13 +107,15 @@ typedef struct SMqConsumeCbParam { tmq_t* tmq; SMqClientVg* pVg; tmq_message_t** retMsg; + tsem_t rspSem; } SMqConsumeCbParam; -typedef struct SMqSubscribeCbParam { - tmq_t* tmq; - tsem_t rspSem; - tmq_resp_err_t rspErr; -} SMqSubscribeCbParam; +typedef struct SMqCommitCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + int32_t async; + tsem_t rspSem; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); @@ -116,7 +123,7 @@ tmq_conf_t* tmq_conf_new() { } void tmq_conf_destroy(tmq_conf_t* conf) { - if(conf) free(conf); + if (conf) free(conf); } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { @@ -130,7 +137,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } tmq_list_t* tmq_list_new() { - tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + tmq_list_t* ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); if (ptr == NULL) { return ptr; } @@ -140,7 +147,7 @@ tmq_list_t* tmq_list_new() { } int32_t tmq_list_append(tmq_list_t* ptr, char* src) { - if (ptr->cnt >= ptr->tot-1) return -1; + if (ptr->cnt >= ptr->tot - 1) return -1; ptr->elems[ptr->cnt] = strdup(src); ptr->cnt++; return 0; @@ -153,6 +160,16 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } +int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqCommitCbParam* pParam = (SMqCommitCbParam*) param; + tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + if (pParam->tmq->commit_cb) { + pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); + } + if (!pParam->async) tsem_post(&pParam->rspSem); + return 0; +} + tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = calloc(sizeof(tmq_t), 1); if (pTmq == NULL) { @@ -161,7 +178,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->pTscObj = (STscObj*)conn; pTmq->status = 0; pTmq->pollCnt = 0; - pTmq->epoch = 0; + pTmq->epoch = 0; taosInitRWLatch(&pTmq->lock); strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -173,9 +190,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { - SRequestObj *pRequest = NULL; - int32_t sz = topic_list->cnt; - //destroy ex + SRequestObj* pRequest = NULL; + int32_t sz = topic_list->cnt; + // destroy ex taosArrayDestroy(tmq->clientTopics); tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); @@ -190,34 +207,34 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SName name = {0}; char* dbName = getDbOfConnection(tmq->pTscObj); - tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); + tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); tNameFromString(&name, topicName, T_NAME_TABLE); char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFname == NULL) { - } tNameExtractFullName(&name, topicFname); tscDebug("subscribe topic: %s", topicFname); SMqClientTopic topic = { - .nextVgIdx = 0, - .sql = NULL, - .sqlLen = 0, - .topicId = 0, - .topicName = topicFname, - .vgs = NULL + .nextVgIdx = 0, + .sql = NULL, + .sqlLen = 0, + .topicId = 0, + .topicName = topicFname, + .vgs = NULL }; topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); - taosArrayPush(tmq->clientTopics, &topic); + taosArrayPush(tmq->clientTopics, &topic); /*SMqClientTopic topic = {*/ - /*.*/ + /*.*/ /*};*/ taosArrayPush(req.topicNames, &topicFname); + free(dbName); } - int tlen = tSerializeSCMSubscribeReq(NULL, &req); + int tlen = tSerializeSCMSubscribeReq(NULL, &req); void* buf = malloc(tlen); - if(buf == NULL) { + if (buf == NULL) { goto _return; } @@ -230,13 +247,10 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tscError("failed to malloc sqlObj"); } - SMqSubscribeCbParam param = { - .rspErr = TMQ_RESP_ERR__SUCCESS, - .tmq = tmq - }; + SMqSubscribeCbParam param = {.rspErr = TMQ_RESP_ERR__SUCCESS, .tmq = tmq}; tsem_init(¶m.rspSem, 0, 0); - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->param = ¶m; @@ -251,18 +265,16 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { _return: /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ + /*destroySendMsgInfo(sendInfo);*/ /*}*/ return param.rspErr; } -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { - conf->commit_cb = cb; -} +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { - tmq_t* pTmq = (void*)param; + tmq_t* pTmq = (void*)param; SArray* pArray = taosArrayInit(0, sizeof(SKv)); if (pArray == NULL) { return NULL; @@ -276,12 +288,12 @@ SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { } pMqHb->consumerId = connKey.connId; SArray* clientTopics = pTmq->clientTopics; - int sz = taosArrayGetSize(clientTopics); + int sz = taosArrayGetSize(clientTopics); for (int i = 0; i < sz; i++) { SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); /*if (pCTopic->vgId == -1) {*/ - /*pMqHb->status = 1;*/ - /*break;*/ + /*pMqHb->status = 1;*/ + /*break;*/ /*}*/ } kv.value = pMqHb; @@ -291,24 +303,11 @@ SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { return pArray; } -tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); - if (pTmq == NULL) { - return NULL; - } - strcpy(pTmq->groupId, conf->groupId); - strcpy(pTmq->clientId, conf->clientId); - pTmq->pTscObj = (STscObj*)conn; - pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; - - return pTmq; -} - -TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { - STscObj *pTscObj = (STscObj*)taos; - SRequestObj *pRequest = NULL; - SQueryNode *pQueryNode = NULL; - char *pStr = NULL; +TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { + STscObj* pTscObj = (STscObj*)taos; + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + char* pStr = NULL; terrno = TSDB_CODE_SUCCESS; if (taos == NULL || topicName == NULL || sql == NULL) { @@ -334,17 +333,18 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); - SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo* ) pQueryNode; + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*)pQueryNode; pQueryStmtInfo->info.continueQuery = true; // todo check for invalid sql statement and return with error code - SSchema *schema = NULL; + SSchema* schema = NULL; int32_t numOfCols = 0; - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), + _return); pStr = qDagToString(pRequest->body.pDag); - if(pStr == NULL) { + if (pStr == NULL) { goto _return; } @@ -352,24 +352,24 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, // The topic should be related to a database that the queried table is belonged to. SName name = {0}; - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName); + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName); - tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, topicName, T_NAME_TABLE); char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; tNameExtractFullName(&name, topicFname); SCMCreateTopicReq req = { - .name = (char*) topicFname, - .igExists = 1, - .physicalPlan = (char*) pStr, - .sql = (char*) sql, - .logicalPlan = "no logic plan", + .name = (char*)topicFname, + .igExists = 1, + .physicalPlan = (char*)pStr, + .sql = (char*)sql, + .logicalPlan = "no logic plan", }; - int tlen = tSerializeSCMCreateTopicReq(NULL, &req); + int tlen = tSerializeSCMCreateTopicReq(NULL, &req); void* buf = malloc(tlen); if (buf == NULL) { goto _return; @@ -379,11 +379,11 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, tSerializeSCMCreateTopicReq(&abuf, &req); /*printf("formatted: %s\n", dagStr);*/ - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; pRequest->type = TDMT_MND_CREATE_TOPIC; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); @@ -393,7 +393,7 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, _return: qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ + /*destroySendMsgInfo(sendInfo);*/ /*}*/ if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { @@ -403,7 +403,7 @@ _return: return pRequest; } -static char *formatTimestamp(char *buf, int64_t val, int precision) { +static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; int32_t ms = 0; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -439,7 +439,7 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { } } - struct tm *ptm = localtime(&tt); + struct tm* ptm = localtime(&tt); size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); if (precision == TSDB_TIME_PRECISION_NANO) { @@ -453,43 +453,35 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { - if (code == -1) { - printf("msg discard\n"); - free(param); - return 0; +void tmqShowMsg(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; + + static bool noPrintSchema; + char pBuf[128]; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + int32_t colNum = pRsp->schemas->nCols; + if (!noPrintSchema) { + printf("|"); + for (int32_t i = 0; i < colNum; i++) { + if (i == 0) + printf(" %25s |", pRsp->schemas->pSchema[i].name); + else + printf(" %15s |", pRsp->schemas->pSchema[i].name); + } + printf("\n"); + printf("===============================================\n"); + noPrintSchema = true; } - char pBuf[128]; - SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; - SMqClientVg* pVg = pParam->pVg; - SMqConsumeRsp rsp = {0}; - tDecodeSMqConsumeRsp(pMsg->pData, &rsp); - if (rsp.numOfTopics == 0) { - /*printf("no data\n");*/ - free(param); - return 0; - } - int32_t colNum = rsp.schemas->nCols; - pVg->currentOffset = rsp.rspOffset; - /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ - /*printf("-----msg begin----\n");*/ - printf("|"); - for (int32_t i = 0; i < colNum; i++) { - if (i == 0) printf(" %25s |", rsp.schemas->pSchema[i].name); - else printf(" %15s |", rsp.schemas->pSchema[i].name); - } - printf("\n"); - printf("===============================================\n"); - int32_t sz = taosArrayGetSize(rsp.pBlockData); + int32_t sz = taosArrayGetSize(pRsp->pBlockData); for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); - int32_t rows = pDataBlock->info.rows; + SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i); + int32_t rows = pDataBlock->info.rows; for (int32_t j = 0; j < rows; j++) { printf("|"); for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch(pColInfoData->info.type) { + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); printf(" %25s |", pBuf); @@ -503,15 +495,42 @@ int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { printf("\n"); } } - tDeleteSMqConsumeRsp(&rsp); - free(param); +} + +int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; + SMqClientVg* pVg = pParam->pVg; + if (code != 0) { + printf("msg discard\n"); + tsem_post(&pParam->rspSem); + return 0; + } + + SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp)); + if (pRsp == NULL) { + tsem_post(&pParam->rspSem); + return -1; + } + tDecodeSMqConsumeRsp(pMsg->pData, pRsp); + printf("rsp %ld %ld\n", pRsp->committedOffset, pRsp->rspOffset); + if (pRsp->numOfTopics == 0) { + /*printf("no data\n");*/ + free(pRsp); + tsem_post(&pParam->rspSem); + return 0; + } + *pParam->retMsg = (tmq_message_t*)pRsp; + pVg->currentOffset = pRsp->rspOffset; + /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ + /*printf("-----msg begin----\n");*/ + tsem_post(&pParam->rspSem); /*printf("\n-----msg end------\n");*/ return 0; } -int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - tmq_t* tmq = pParam->tmq; + tmq_t* tmq = pParam->tmq; if (code != 0) { printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); if (pParam->wait) { @@ -520,15 +539,15 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } tscDebug("tmq ask ep cb called"); - bool set = false; + bool set = false; SMqCMGetSubEpRsp rsp; tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); int32_t sz = taosArrayGetSize(rsp.topics); // TODO: lock - /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ - /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ + /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ if (rsp.epoch != tmq->epoch) { - //TODO + // TODO if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); for (int32_t i = 0; i < sz; i++) { @@ -540,12 +559,7 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { for (int32_t j = 0; j < vgSz; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); SMqClientVg clientVg = { - .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet - }; + .pollCnt = 0, .committedOffset = -1, .currentOffset = -1, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet}; taosArrayPush(topic.vgs, &clientVg); set = true; } @@ -562,66 +576,67 @@ int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { tsem_post(&tmq->rspSem); } tDeleteSMqCMGetSubEpRsp(&rsp); - free(pParam); return 0; } int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { - int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* buf = malloc(tlen); - if (buf == NULL) { - goto END; - tscError("failed to malloc get subscribe ep buf"); - } - buf->consumerId = htobe64(tmq->consumerId); - strcpy(buf->cgroup, tmq->groupId); - - SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); - if (pRequest == NULL) { - goto END; - tscError("failed to malloc subscribe ep request"); - } + int32_t tlen = sizeof(SMqCMGetSubEpReq); + SMqCMGetSubEpReq* buf = malloc(tlen); + if (buf == NULL) { + tscError("failed to malloc get subscribe ep buf"); + goto END; + } + buf->consumerId = htobe64(tmq->consumerId); + strcpy(buf->cgroup, tmq->groupId); - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); + if (pRequest == NULL) { + tscError("failed to malloc subscribe ep request"); + goto END; + } - SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); - if (pParam == NULL) { - goto END; - } - pParam->tmq = tmq; - pParam->wait = wait; + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmq_ask_ep_cb; + SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + tscError("failed to malloc subscribe param"); + goto END; + } + pParam->tmq = tmq; + pParam->wait = wait; - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqAskEpCb; - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); END: - if (wait) tsem_wait(&tmq->rspSem); - return 0; + if (wait) tsem_wait(&tmq->rspSem); + return 0; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, SMqClientVg** ppVg) { +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, + SMqClientVg* pVg) { SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); if (pReq == NULL) { return NULL; } pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); pReq->blockingTime = blocking_time; pReq->consumerId = tmq->consumerId; strcpy(pReq->cgroup, tmq->groupId); - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - strcpy(pReq->topic, pTopic->topicName); - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - pReq->offset = pVg->currentOffset+1; - *ppVg = pVg; + if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { + pReq->offset = pVg->currentOffset; + } else { + pReq->offset = pVg->currentOffset + 1; + } pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -632,10 +647,11 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_message = NULL; int64_t status = atomic_load_64(&tmq->status); - tmqAsyncAskEp(tmq, taosArrayGetSize(tmq->clientTopics)); + tmqAsyncAskEp(tmq, status == 0); - /*if (blocking_time < 0) blocking_time = 500;*/ - blocking_time = 1; + if (blocking_time < 0) blocking_time = 1; + if (blocking_time > 1000) blocking_time = 1000; + /*blocking_time = 1;*/ if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); @@ -648,8 +664,10 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { return NULL; } - SMqClientVg* pVg = NULL; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, &pVg); + tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); if (pReq == NULL) { usleep(blocking_time * 1000); return NULL; @@ -663,14 +681,15 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { param->tmq = tmq; param->retMsg = &tmq_message; param->pVg = pVg; + tsem_init(¶m->rspSem, 0, 0); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) }; + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; sendInfo->param = param; - sendInfo->fp = tmq_poll_cb_inner; + sendInfo->fp = tmqPollCb; /*printf("req offset: %ld\n", pReq->offset);*/ @@ -678,34 +697,92 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); tmq->pollCnt++; - usleep(blocking_time * 1000); + tsem_wait(¶m->rspSem); + tsem_destroy(¶m->rspSem); + free(param); + + if (tmq_message == NULL) { + usleep(blocking_time * 1000); + } return tmq_message; /*tsem_wait(&pRequest->body.rspSem);*/ /*if (body != NULL) {*/ - /*destroySendMsgInfo(body);*/ + /*destroySendMsgInfo(body);*/ /*}*/ /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ - /*pRequest->code = terrno;*/ + /*pRequest->code = terrno;*/ /*}*/ /*return pRequest;*/ } tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - SMqConsumeReq req = {0}; + + if (tmq_topic_vgroup_list != NULL) { + //TODO + } + + //TODO: change semaphore to gate + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + SMqCommitCbParam *pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + continue; + } + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->async = async; + if (!async) tsem_init(&pParam->rspSem, 0, 0); + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + + if (!async) tsem_wait(&pParam->rspSem); + } + } + return 0; } void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + tDeleteSMqConsumeRsp(pRsp); + free(tmq_message); } +#if 0 +tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { + tmq_t* pTmq = malloc(sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + strcpy(pTmq->groupId, conf->groupId); + strcpy(pTmq->clientId, conf->clientId); + pTmq->pTscObj = (STscObj*)conn; + pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; + return pTmq; +} + + static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); tfree(pMsgBody->msgInfo.pData); tfree(pMsgBody); } +#endif diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 7ef06397f0..b7ee64e19f 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -583,7 +583,7 @@ TEST(testCase, create_topic_ctb_Test) { taos_free_result(pRes); char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql)); + pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql)); taos_free_result(pRes); taos_close(pConn); } @@ -607,7 +607,7 @@ TEST(testCase, create_topic_stb_Test) { taos_free_result(pRes); char* sql = "select * from st1"; - pRes = taos_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); taos_free_result(pRes); taos_close(pConn); } @@ -633,11 +633,11 @@ TEST(testCase, tmq_subscribe_ctb_Test) { while (1) { tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + tmq_message_destroy(msg); //printf("get msg\n"); //if (msg == NULL) break; } } -#endif TEST(testCase, tmq_subscribe_stb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -656,11 +656,18 @@ TEST(testCase, tmq_subscribe_stb_Test) { tmq_list_t* topic_list = tmq_list_new(); tmq_list_append(topic_list, "test_stb_topic_1"); tmq_subscribe(tmq, topic_list); - + + int cnt = 1; while (1) { tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + if (msg == NULL) continue; + tmqShowMsg(msg); + if (cnt++ % 10 == 0){ + tmq_commit(tmq, NULL, 0); + } + //tmq_commit(tmq, NULL, 0); + tmq_message_destroy(msg); //printf("get msg\n"); - //if (msg == NULL) break; } } @@ -669,6 +676,7 @@ TEST(testCase, tmq_consume_Test) { TEST(testCase, tmq_commit_TEST) { } +#endif #if 0 TEST(testCase, projection_query_tables) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a9a633cf63..ab9a1cb5ea 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -633,7 +633,7 @@ typedef struct SMqConsumerTopic { } SMqConsumerTopic; static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, - SMqSubscribeObj* pSub) { + SMqSubscribeObj* pSub, int64_t* oldConsumerId) { SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic)); if (pCTopic == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -646,6 +646,7 @@ static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqT int32_t unassignedVgSz = taosArrayGetSize(pSub->unassignedVg); if (unassignedVgSz > 0) { SMqConsumerEp* pCEp = taosArrayPop(pSub->unassignedVg); + *oldConsumerId = pCEp->consumerId; pCEp->consumerId = consumerId; taosArrayPush(pCTopic->pVgInfo, &pCEp->vgId); taosArrayPush(pSub->assigned, pCEp); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 47bdefea3d..bf138d335b 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -48,7 +48,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub); + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub, + int64_t oldConsumerId); int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -166,7 +167,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { if (pCEp->lastConsumerHbTs != -1 && currentTs - pCEp->lastConsumerHbTs > MND_SUBSCRIBE_REBALANCE_MS) { // put consumer into lostConsumer taosArrayPush(pSub->lostConsumer, pCEp); - // put vg into unassgined + // put vg into unassigned taosArrayPush(pSub->unassignedVg, pCEp); // remove from assigned // TODO: swap with last one, reduce size and reset i @@ -202,6 +203,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { for (int32_t i = 0; i < sz; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx); SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); + int64_t oldConsumerId = pCEp->consumerId; pCEp->consumerId = consumerId; taosArrayPush(pSub->assigned, pCEp); pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); @@ -222,7 +224,9 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { req.logicalPlan = pTopic->logicalPlan; req.physicalPlan = pTopic->physicalPlan; req.qmsg = pCEp->qmsg; + req.oldConsumerId = oldConsumerId; req.newConsumerId = consumerId; + req.vgId = pCEp->vgId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *buf = malloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { @@ -237,6 +241,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { tEncodeSMqSetCVgReq(&abuf, &req); // persist msg + // TODO: no need for txn STransAction action = {0}; action.epSet = pCEp->epSet; action.pCont = buf; @@ -303,13 +308,12 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas ASSERT(CEp.vgId == pVgroup->vgId); CEp.qmsg = strdup(pTaskInfo->msg->msg); taosArrayPush(unassignedVg, &CEp); - //TODO: free taskInfo + // TODO: free taskInfo taosArrayDestroy(pArray); /*SEpSet *pEpSet = &plan->execNode.epset;*/ /*pEpSet->inUse = 0;*/ /*addEpIntoEpSet(pEpSet, "localhost", 6030);*/ - } /*qDestroyQueryDag(pDag);*/ @@ -317,14 +321,15 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp) { + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp, + int64_t oldConsumerId) { int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); for (int32_t i = 0; i < sz; i++) { int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i); SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SMqSetCVgReq req = { .vgId = vgId, - .oldConsumerId = -1, + .oldConsumerId = oldConsumerId, .newConsumerId = pConsumer->consumerId, }; strcpy(req.cgroup, pConsumer->cgroup); @@ -459,10 +464,6 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc return 0; } -static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { - return 0; -} - static char *mndMakeSubscribeKey(char *cgroup, char *topicName) { char *key = malloc(TSDB_SHOW_SUBQUERY_LEN); if (key == NULL) { @@ -642,7 +643,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { free(key); // set unassigned vg if (mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg) < 0) { - //TODO: free memory + // TODO: free memory return -1; } // TODO: disable alter @@ -650,7 +651,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } taosArrayPush(pSub->availConsumer, &consumerId); - SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); + int64_t oldConsumerId; + SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub, &oldConsumerId); taosArrayPush(pConsumer->topics, pConsumerTopic); if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { @@ -658,7 +660,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); SMqConsumerEp *pCEp = taosArrayGetLast(pSub->assigned); if (pCEp->vgId == vgId) { - if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp) < 0) { + if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp, oldConsumerId) < 0) { // TODO return -1; } diff --git a/source/dnode/vnode/src/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h index ef71d8bf14..3bf9bb7138 100644 --- a/source/dnode/vnode/src/inc/tqMetaStore.h +++ b/source/dnode/vnode/src/inc/tqMetaStore.h @@ -40,6 +40,7 @@ int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); // delete committed kv pair // notice that a delete action still needs to be committed int32_t tqHandleDel(STqMetaStore*, int64_t key); +int32_t tqHandlePurge(STqMetaStore*, int64_t key); int32_t tqHandleCommit(STqMetaStore*, int64_t key); int32_t tqHandleAbort(STqMetaStore*, int64_t key); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 139120a46d..4e52ecd858 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,17 +25,6 @@ // handle management message // -int tqGroupSSize(const STqGroup* pGroup); -int tqTopicSSize(); -int tqItemSSize(); - -void* tqSerializeListHandle(STqList* listHandle, void* ptr); -void* tqSerializeTopic(STqTopic* pTopic, void* ptr); -void* tqSerializeItem(STqMsgItem* pItem, void* ptr); - -const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); -const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); - int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); if (old == 1) return 0; @@ -88,177 +77,6 @@ void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(STqMsgHead* pMsg) { - // TODO - return pMsg->protoVer == 0; -} - -static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { - // clean old item and move forward - int32_t consumeOffset = pAck->consumeOffset; - int idx = consumeOffset % TQ_BUFFER_SIZE; - ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor); - tfree(pTopic->buffer[idx].content); - if (1 /* TODO: need to launch new query */) { - STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); - if (pNewQuery == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - // TODO: lock executor - // TODO: read from wal and assign to src - /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/ - /*pNewQuery->exec->src = 0;*/ - /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/ - /*pNewQuery->next = *ppQuery;*/ - /**ppQuery = pNewQuery;*/ - } - return 0; -} - -static int tqAck(STqGroup* pGroup, STqAcks* pAcks) { - int32_t ackNum = pAcks->ackNum; - STqOneAck* acks = pAcks->acks; - // double ptr for acks and list - int i = 0; - STqList* node = pGroup->head; - int ackCnt = 0; - STqQueryMsg* pQuery = NULL; - while (i < ackNum && node->next) { - if (acks[i].topicId == node->next->topic.topicId) { - ackCnt++; - tqAckOneTopic(&node->next->topic, &acks[i], &pQuery); - } else if (acks[i].topicId < node->next->topic.topicId) { - i++; - } else { - node = node->next; - } - } - if (pQuery) { - // post message - } - return ackCnt; -} - -static int tqCommitGroup(STqGroup* pGroup) { - // persist modification into disk - return 0; -} - -int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) { - // create in disk - STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup)); - if (pGroup == NULL) { - // TODO - return -1; - } - *ppGroup = pGroup; - memset(pGroup, 0, sizeof(STqGroup)); - - pGroup->topicList = tdListNew(sizeof(STqTopic)); - if (pGroup->topicList == NULL) { - free(pGroup); - return -1; - } - *ppGroup = pGroup; - - return 0; -} - -STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId); - if (pGroup == NULL) { - int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup); - if (code < 0) { - // TODO - return NULL; - } - tqHandleMovePut(pTq->tqMeta, cId, pGroup); - } - ASSERT(pGroup); - - return pGroup; -} - -int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - // TODO - return 0; -} - -int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - // delete from disk - return 0; -} - -static int tqFetch(STqGroup* pGroup, STqConsumeRsp** pRsp) { - STqList* pHead = pGroup->head; - STqList* pNode = pHead; - int totSize = 0; - int numOfMsgs = 0; - // TODO: make it a macro - int sizeLimit = 4 * 1024; - - void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + sizeLimit); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - *pRsp = ptr; - STqMsgContent* buffer = (*pRsp)->msgs; - - // iterate the list to get msgs of all topics - // until all topic iterated or msgs over sizeLimit - while (pNode->next) { - pNode = pNode->next; - STqTopic* pTopic = &pNode->topic; - int idx = pTopic->nextConsumeOffset % TQ_BUFFER_SIZE; - if (pTopic->buffer[idx].content != NULL && pTopic->buffer[idx].offset == pTopic->nextConsumeOffset) { - totSize += pTopic->buffer[idx].size; - if (totSize > sizeLimit) { - void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + totSize); - if (ptr == NULL) { - totSize -= pTopic->buffer[idx].size; - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // return msgs already copied - break; - } - *pRsp = ptr; - break; - } - *((int64_t*)buffer) = pTopic->topicId; - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = pTopic->buffer[idx].size; - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); - numOfMsgs++; - if (totSize > sizeLimit) { - break; - } - } - } - (*pRsp)->bodySize = totSize; - return numOfMsgs; -} - -STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } - -int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) { - if (tqQueryExecuting(bufItem->status)) { - return 0; - } - bufItem->status = 1; - // load data from wal or buffer pool - // put into exec - // send exec into non blocking queue - // when query finished, put into buffer pool - return 0; -} - -/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ -/*return 0;*/ -/*}*/ - int tqPushMsg(STQ* pTq, void* p, int64_t version) { // add reference // judge and launch new query @@ -270,218 +88,6 @@ int tqCommit(STQ* pTq) { return 0; } -int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) { - int code; - memset(pTopic->buffer, 0, sizeof(pTopic->buffer)); - // launch query - for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) { - int pos = i % TQ_BUFFER_SIZE; - code = tqSendLaunchQuery(&pTopic->buffer[pos], offset); - if (code < 0) { - // TODO: error handling - } - } - // set offset - pTopic->nextConsumeOffset = offset; - pTopic->floatingCursor = offset; - return 0; -} - -STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) { - // TODO - return NULL; -} - -int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { - int code; - int64_t clientId = pMsg->head.clientId; - int64_t topicId = pMsg->topicId; - int64_t offset = pMsg->offset; - STqGroup* gHandle = tqGetGroup(pTq, clientId); - if (gHandle == NULL) { - // client not connect - return -1; - } - STqTopic* topicHandle = tqFindTopic(gHandle, topicId); - if (topicHandle == NULL) { - return -1; - } - if (pMsg->offset == topicHandle->nextConsumeOffset) { - return 0; - } - // TODO: check log last version - - code = tqBufferSetOffset(topicHandle, offset); - if (code < 0) { - // set error code - return -1; - } - - return 0; -} - -// temporary -int tqProcessCMsg(STQ* pTq, STqConsumeReq* pMsg, STqRspHandle* pRsp) { - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - terrno = TSDB_CODE_TQ_GROUP_NOT_SET; - return -1; - } - pGroup->rspHandle.handle = pRsp->handle; - pGroup->rspHandle.ahandle = pRsp->ahandle; - - return 0; -} - -int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { - STqConsumeReq* pMsg = pReq->pCont; - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - terrno = TSDB_CODE_TQ_GROUP_NOT_SET; - return -1; - } - - SList* topicList = pGroup->topicList; - - int totSize = 0; - int numOfMsgs = 0; - int sizeLimit = 4096; - - STqConsumeRsp* pCsmRsp = (*pRsp)->pCont; - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - (*pRsp)->pCont = ptr; - - SListIter iter; - tdListInitIter(topicList, &iter, TD_LIST_FORWARD); - - STqMsgContent* buffer = NULL; - SArray* pArray = taosArrayInit(0, sizeof(void*)); - - SListNode* pn; - while ((pn = tdListNext(&iter)) != NULL) { - STqTopic* pTopic = *(STqTopic**)pn->data; - int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; - STqMsgItem* pItem = &pTopic->buffer[idx]; - if (pItem->content != NULL && pItem->offset == pTopic->floatingCursor) { - if (pItem->status == TQ_ITEM_READY) { - // if has data - totSize += pTopic->buffer[idx].size; - if (totSize > sizeLimit) { - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + totSize); - if (ptr == NULL) { - totSize -= pTopic->buffer[idx].size; - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // return msgs already copied - break; - } - (*pRsp)->pCont = ptr; - break; - } - *((int64_t*)buffer) = htobe64(pTopic->topicId); - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = htobe64(pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); - numOfMsgs++; - if (totSize > sizeLimit) { - break; - } - } else if (pItem->status == TQ_ITEM_PROCESS) { - // if not have data but in process - - } else if (pItem->status == TQ_ITEM_EMPTY) { - // if not have data and not in process - int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_EMPTY, TQ_ITEM_PROCESS); - if (old != TQ_ITEM_EMPTY) { - continue; - } - pItem->offset = pTopic->floatingCursor; - taosArrayPush(pArray, &pItem); - } else { - ASSERT(0); - } - } - } - - if (numOfMsgs > 0) { - // set code and other msg - rpcSendResponse(*pRsp); - } else { - // most recent data has been fetched - - // enable timer for blocking wait - // once new data written when waiting, launch query and rsp - } - - // fetched a num of msgs, rpc response - for (int i = 0; i < pArray->size; i++) { - STqMsgItem* pItem = taosArrayGet(pArray, i); - - // read from wal - void* raw = NULL; - /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ - /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ - /*if (code < 0) {*/ - // TODO: error - /*}*/ - // get msgType - // if submitblk - pItem->executor->assign(pItem->executor->runtimeEnv, raw); - SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); - pItem->content = content; - // if other type, send just put into buffer - /*pItem->content = raw;*/ - - int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); - ASSERT(old == TQ_ITEM_PROCESS); - } - taosArrayDestroy(pArray); - - return 0; -} - -#if 0 -int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { - if (!tqProtoCheck((STqMsgHead*)pMsg)) { - // proto version invalid - return -1; - } - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - // client not connect - return -1; - } - if (pMsg->acks.ackNum != 0) { - if (tqAck(pGroup, &pMsg->acks) != 0) { - // ack not success - return -1; - } - } - - STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - - if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) { - // fetch error - return -1; - } - - // judge and launch new query - /*if (tqSendLaunchQuery(gHandle)) {*/ - // launch query error - /*return -1;*/ - /*}*/ - return 0; -} -#endif - int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { int32_t num = taosArrayGetSize(pConsumer->topics); int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + @@ -535,138 +141,6 @@ const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHan return NULL; } -#if 0 -int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { - // calculate size - int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); - if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sz); - if (tmpPtr == NULL) { - free(*ppHead); - // TODO: memory err - return -1; - } - *ppHead = tmpPtr; - (*ppHead)->ssize = sz; - } - void* ptr = (*ppHead)->content; - // do serialization - *(int64_t*)ptr = pGroup->clientId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pGroup->cgId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = pGroup->topicNum; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if (pGroup->topicNum > 0) { - tqSerializeListHandle(pGroup->head, ptr); - } - return 0; -} - -void* tqSerializeListHandle(STqList* listHandle, void* ptr) { - STqList* node = listHandle; - ASSERT(node != NULL); - while (node) { - ptr = tqSerializeTopic(&node->topic, ptr); - node = node->next; - } - return ptr; -} - -void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { - *(int64_t*)ptr = pTopic->nextConsumeOffset; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pTopic->topicId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - /**(int32_t*)ptr = pTopic->head;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - /**(int32_t*)ptr = pTopic->tail;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqSerializeItem(&pTopic->buffer[i], ptr); - } - return ptr; -} - -void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) { - // TODO: do we need serialize this? - // mainly for executor - return ptr; -} - -const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) { - STqGroup* gHandle = *ppGroup; - const void* ptr = pHead->content; - gHandle->clientId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - gHandle->cgId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - gHandle->ahandle = NULL; - gHandle->topicNum = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - gHandle->head = NULL; - STqList* node = gHandle->head; - for (int i = 0; i < gHandle->topicNum; i++) { - if (gHandle->head == NULL) { - if ((node = malloc(sizeof(STqList))) == NULL) { - // TODO: error - return NULL; - } - node->next = NULL; - ptr = tqDeserializeTopic(ptr, &node->topic); - gHandle->head = node; - } else { - node->next = malloc(sizeof(STqList)); - if (node->next == NULL) { - // TODO: error - return NULL; - } - node->next->next = NULL; - ptr = tqDeserializeTopic(ptr, &node->next->topic); - node = node->next; - } - } - return ptr; -} - -const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { - const void* ptr = pBytes; - topic->nextConsumeOffset = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - topic->topicId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - /*topic->head = *(int32_t*)ptr;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - /*topic->tail = *(int32_t*)ptr;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqDeserializeItem(ptr, &topic->buffer[i]); - } - return ptr; -} - -const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; } - -// TODO: make this a macro -int tqGroupSSize(const STqGroup* gHandle) { - return sizeof(int64_t) * 2 // cId + cgId - + sizeof(int32_t) // topicNum - + gHandle->topicNum * tqTopicSSize(); -} - -// TODO: make this a macro -int tqTopicSSize() { - return sizeof(int64_t) * 2 // nextConsumeOffset + topicId - + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqItemSSize(); -} - -int tqItemSSize() { - // TODO: do this need serialization? - // mainly for executor - return 0; -} -#endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; @@ -675,7 +149,6 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { int64_t fetchOffset = pReq->offset; int64_t blockingTime = pReq->blockingTime; - int rspLen = 0; SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); @@ -694,11 +167,25 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (strcmp(pTopic->topicName, pReq->topic) != 0) { continue; } + + if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { + pTopic->committedOffset = pReq->offset; + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = 0; + rpcSendResponse(pMsg); + return 0; + } + + if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { + pTopic->committedOffset = pReq->offset-1; + } + rsp.committedOffset = pTopic->committedOffset; rsp.reqOffset = pReq->offset; rsp.skipLogNum = 0; - if (fetchOffset == -1) { + if (fetchOffset <= pTopic->committedOffset) { fetchOffset = pTopic->committedOffset + 1; } int8_t pos; @@ -802,11 +289,22 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { - SMqSetCVgReq req; + SMqSetCVgReq req = {0}; tDecodeSMqSetCVgReq(msg, &req); - STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); + + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); if (pConsumer == NULL) { - return -1; + pConsumer = calloc(sizeof(STqConsumerHandle), 1); + if (pConsumer == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + } else { + tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.newConsumerId); + tqHandlePurge(pTq->tqMeta, req.oldConsumerId); + terrno = TSDB_CODE_SUCCESS; + return 0; } strcpy(pConsumer->cgroup, req.cgroup); pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); @@ -819,9 +317,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { return -1; } strcpy(pTopic->topicName, req.topicName); - pTopic->sql = strdup(req.sql); - pTopic->logicalPlan = strdup(req.logicalPlan); - pTopic->physicalPlan = strdup(req.physicalPlan); + pTopic->sql = req.sql; + pTopic->logicalPlan = req.logicalPlan; + pTopic->physicalPlan = req.physicalPlan; pTopic->committedOffset = -1; pTopic->currentOffset = -1; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 57e20010e3..d220966ba6 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -584,12 +584,30 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if (pNode->handle.valueInTxn) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn) { + pMeta->pDeleter(pNode->handle.valueInTxn); + } - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; + pNode->handle.valueInTxn = TQ_DELETE_TOKEN; + tqLinkUnpersist(pMeta, pNode); + return 0; + } + } else { + pNode = pNode->next; + } + } + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; +} + +int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) { + int64_t bucketKey = key & TQ_BUCKET_MASK; + STqMetaList* pNode = pMeta->bucket[bucketKey]; + while (pNode) { + if (pNode->handle.key == key) { + pNode->handle.valueInUse = TQ_DELETE_TOKEN; tqLinkUnpersist(pMeta, pNode); return 0; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 326d99ddbb..faf3171d76 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -16,6 +16,7 @@ #include "tq.h" #include "vnd.h" +#if 0 int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_VND_MQ_SET_CUR: @@ -26,6 +27,7 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { } return 0; } +#endif int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { SRpcMsg *pMsg; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 96d0c6c385..41fff4d228 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -169,10 +169,17 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } - ASSERT(pRead->pHead->head.version == ver); + if (pRead->pHead->head.version != ver) { + /*wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver);*/ + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } code = walValidBodyCksum(pRead->pHead); if (code != 0) { + /*wError("unexpected wal log version: checksum not passed");*/ + pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } From 174378ad2b09b2224308f8920a184e3e7e326444 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 29 Jan 2022 09:56:05 +0800 Subject: [PATCH 38/42] add demo for tmq --- CMakeLists.txt | 1 + example/CMakeLists.txt | 16 +++ example/src/tmq.c | 168 +++++++++++++++++++++++++++ include/client/taos.h | 3 +- source/client/src/tmq.c | 16 ++- source/client/test/clientTests.cpp | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 12 +- 7 files changed, 209 insertions(+), 9 deletions(-) create mode 100644 example/CMakeLists.txt create mode 100644 example/src/tmq.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0383cc8aed..fd542966cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) add_subdirectory(tests) +add_subdirectory(example) # docs add_subdirectory(docs) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt new file mode 100644 index 0000000000..71a9f9f4c5 --- /dev/null +++ b/example/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src TMQ_DEMO_SRC) + +add_executable(tmq ${TMQ_DEMO_SRC}) +target_link_libraries( + tmq + PUBLIC taos + #PUBLIC util + #PUBLIC common + #PUBLIC os +) +target_include_directories( + tmq + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) diff --git a/example/src/tmq.c b/example/src/tmq.c new file mode 100644 index 0000000000..c1c6677b02 --- /dev/null +++ b/example/src/tmq.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include "taos.h" + +static int running = 1; +static void msg_process(tmq_message_t* message) { + tmqShowMsg(message); +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu2 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + + char* sql = "select * from st1"; + pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ + taos_close(pConn); + return 0; +} + +tmq_t* build_consumer() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + return tmq; + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_subscribe(tmq, topic_list); + return NULL; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + return topic_list; +} + +void basic_consume_loop(tmq_t *tmq, + tmq_list_t *topics) { + tmq_resp_err_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + printf("subscribe err\n"); + return; + } + + while (running) { + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); + if (tmq) { + msg_process(tmqmessage); + tmq_message_destroy(tmqmessage); + } + } + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void sync_consume_loop(tmq_t *rk, + tmq_list_t *topics) { + static const int MIN_COMMIT_COUNT = 1000; + + int msg_count = 0; + tmq_resp_err_t err; + + if ((err = tmq_subscribe(rk, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + return; + } + + while (running) { + tmq_message_t *tmqmessage = tmq_consumer_poll(rk, 500); + if (tmqmessage) { + msg_process(tmqmessage); + tmq_message_destroy(tmqmessage); + + if ((++msg_count % MIN_COMMIT_COUNT) == 0) + tmq_commit(rk, NULL, 0); + } + } + + err = tmq_consumer_close(rk); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main() { + int code; + code = init_env(); + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); +} diff --git a/include/client/taos.h b/include/client/taos.h index ac2893651d..7c5cc56b20 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -218,6 +218,7 @@ DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, char *); DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT void tmq_message_destroy(tmq_message_t* tmq_message); +DLL_EXPORT const char* tmq_err2str(tmq_resp_err_t); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); @@ -226,8 +227,8 @@ DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); -#if 0 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t* tmq); +#if 0 DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups); #endif diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d529662008..facc8c7216 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -348,7 +348,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i goto _return; } - printf("%s\n", pStr); + /*printf("%s\n", pStr);*/ // The topic should be related to a database that the queried table is belonged to. SName name = {0}; @@ -501,7 +501,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; SMqClientVg* pVg = pParam->pVg; if (code != 0) { - printf("msg discard\n"); + /*printf("msg discard\n");*/ tsem_post(&pParam->rspSem); return 0; } @@ -512,7 +512,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return -1; } tDecodeSMqConsumeRsp(pMsg->pData, pRsp); - printf("rsp %ld %ld\n", pRsp->committedOffset, pRsp->rspOffset); + /*printf("rsp %ld %ld\n", pRsp->committedOffset, pRsp->rspOffset);*/ if (pRsp->numOfTopics == 0) { /*printf("no data\n");*/ free(pRsp); @@ -766,6 +766,16 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { free(tmq_message); } +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { + return TMQ_RESP_ERR__SUCCESS; +} + +const char* tmq_err2str(tmq_resp_err_t err) { + if (err == TMQ_RESP_ERR__SUCCESS) { + return "success"; + } + return "fail"; +} #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* pTmq = malloc(sizeof(tmq_t)); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b7ee64e19f..553dacafdd 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -638,6 +638,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) { //if (msg == NULL) break; } } +#endif TEST(testCase, tmq_subscribe_stb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -676,7 +677,6 @@ TEST(testCase, tmq_consume_Test) { TEST(testCase, tmq_commit_TEST) { } -#endif #if 0 TEST(testCase, projection_query_tables) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ab9a1cb5ea..75f90df658 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -583,19 +583,23 @@ static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { pSub->availConsumer = NULL; } if (pSub->assigned) { - taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); + //taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->assigned); pSub->assigned = NULL; } if (pSub->unassignedVg) { - taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); + //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->unassignedVg); pSub->unassignedVg = NULL; } if (pSub->idleConsumer) { - taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + //taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->idleConsumer); pSub->idleConsumer = NULL; } if (pSub->lostConsumer) { - taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + //taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->lostConsumer); pSub->lostConsumer = NULL; } } From 2da85fb9dc8ac505c2fa0757badea2acf5fd749a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 29 Jan 2022 20:14:33 +0800 Subject: [PATCH 39/42] clear unused code in tq --- source/dnode/vnode/inc/tq.h | 123 ------------------------------------ 1 file changed, 123 deletions(-) diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index faaf769e1a..2aceaeb016 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -35,71 +35,8 @@ extern "C" { #endif -typedef struct STqMsgHead { - int32_t protoVer; - int32_t msgType; - int64_t cgId; - int64_t clientId; -} STqMsgHead; - -typedef struct STqOneAck { - int64_t topicId; - int64_t consumeOffset; -} STqOneAck; - -typedef struct STqAcks { - int32_t ackNum; - // should be sorted - STqOneAck acks[]; -} STqAcks; - -typedef struct STqSetCurReq { - STqMsgHead head; - int64_t topicId; - int64_t offset; -} STqSetCurReq; - -typedef struct STqConsumeReq { - STqMsgHead head; - int64_t blockingTime; // milisec - STqAcks acks; -} STqConsumeReq; - -typedef struct STqMsgContent { - int64_t topicId; - int64_t msgLen; - char msg[]; -} STqMsgContent; - -typedef struct STqConsumeRsp { - STqMsgHead head; - int64_t bodySize; - STqMsgContent msgs[]; -} STqConsumeRsp; - -typedef struct STqSubscribeReq { - STqMsgHead head; - int32_t topicNum; - int64_t topic[]; -} STqSubscribeReq; - -typedef struct STqHeartbeatReq { -} STqHeartbeatReq; - -typedef struct STqHeartbeatRsp { -} STqHeartbeatRsp; - #define TQ_BUFFER_SIZE 8 -typedef struct STqExec { - void* runtimeEnv; - SSDataBlock* (*exec)(void* runtimeEnv); - void* (*assign)(void* runtimeEnv, void* inputData); - void (*clear)(void* runtimeEnv); - char* (*serialize)(struct STqExec*); - struct STqExec* (*deserialize)(char*); -} STqExec; - typedef struct STqRspHandle { void* handle; void* ahandle; @@ -107,47 +44,6 @@ typedef struct STqRspHandle { typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; -typedef struct STqTopic STqTopic; - -typedef struct STqBufferItem { - int64_t offset; - // executors are identical but not concurrent - // so there must be a copy in each item - STqExec* executor; - int32_t status; - int64_t size; - void* content; - STqTopic* pTopic; -} STqMsgItem; - -struct STqTopic { - // char* topic; //c style, end with '\0' - // int64_t cgId; - // void* ahandle; - // int32_t head; - // int32_t tail; - int64_t nextConsumeOffset; - int64_t floatingCursor; - int64_t topicId; - void* logReader; - STqMsgItem buffer[TQ_BUFFER_SIZE]; -}; - -typedef struct STqListHandle { - STqTopic topic; - struct STqListHandle* next; -} STqList; - -typedef struct STqGroup { - int64_t clientId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqList* head; - SList* topicList; // SList - STqRspHandle rspHandle; -} STqGroup; - typedef struct STqTaskItem { int8_t status; int64_t offset; @@ -182,11 +78,6 @@ typedef struct STqConsumerHandle { SArray* topics; // SArray } STqConsumerHandle; -typedef struct STqQueryMsg { - STqMsgItem* item; - struct STqQueryMsg* next; -} STqQueryMsg; - typedef struct STqMemRef { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; @@ -305,20 +196,6 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqSetCursor(STQ*, STqSetCurReq* pMsg); - -#if 0 -int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); -int tqSetCursor(STQ*, STqSetCurReq* pMsg); -int tqBufferSetOffset(STqTopic*, int64_t offset); -STqTopic* tqFindTopic(STqGroup*, int64_t topicId); -STqGroup* tqGetGroup(STQ*, int64_t clientId); -STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqRegisterContext(STqGroup*, void* ahandle); -int tqSendLaunchQuery(STqMsgItem*, int64_t offset); -#endif - int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); From 7f84cc1f7954614c8544799cacf8b92c2f46cdef Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 1 Feb 2022 20:30:55 +0800 Subject: [PATCH 40/42] fix valgrind error --- source/util/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index ed936e90f6..6d8dfa51e7 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -9,9 +9,7 @@ target_include_directories( target_link_libraries( util PRIVATE os - PUBLIC zlib PUBLIC lz4_static - PUBLIC api ) if(${BUILD_TEST}) From 573aad5cc14aa6d1f7f3d18f96ff38ec04863bc7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 3 Feb 2022 19:47:27 +0800 Subject: [PATCH 41/42] fix uninitialized --- .gitignore | 2 + example/src/tmq.c | 21 +++++--- include/client/taos.h | 2 +- include/os/osEndian.h | 2 +- include/os/osMath.h | 30 +++++------ source/client/src/tmq.c | 8 +-- source/common/src/ttypes.c | 8 --- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++- source/dnode/mnode/impl/src/mndSubscribe.c | 7 ++- source/dnode/vnode/src/inc/tqInt.h | 61 ++++++++++++++-------- source/dnode/vnode/src/tq/tq.c | 9 +--- source/libs/wal/src/walRead.c | 4 +- source/os/src/osDir.c | 2 + source/util/src/tbuffer.c | 2 + source/util/src/tlog.c | 1 - source/util/src/tpagedfile.c | 17 ++++++ 16 files changed, 112 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index f0067d26ba..b9b5341b06 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ build/ compile_commands.json .cache .ycm_extra_conf.py +.tasks +.vimspector.json .vscode/ .idea/ cmake-build-debug/ diff --git a/example/src/tmq.c b/example/src/tmq.c index c1c6677b02..31bfe4197f 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "taos.h" static int running = 1; @@ -65,7 +66,7 @@ int32_t init_env() { taos_free_result(pRes); - char* sql = "select * from st1"; + const char* sql = "select * from st1"; pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); /*if (taos_errno(pRes) != 0) {*/ /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ @@ -112,14 +113,20 @@ void basic_consume_loop(tmq_t *tmq, printf("subscribe err\n"); return; } - + int32_t cnt = 0; + clock_t startTime = clock(); while (running) { - tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); - if (tmq) { - msg_process(tmqmessage); + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 0); + if (tmqmessage) { + cnt++; + /*msg_process(tmqmessage);*/ tmq_message_destroy(tmqmessage); + } else { + break; } } + clock_t endTime = clock(); + printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC); err = tmq_consumer_close(tmq); if (err) @@ -163,6 +170,6 @@ int main() { code = init_env(); tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); - /*basic_consume_loop(tmq, topic_list);*/ - sync_consume_loop(tmq, topic_list); + basic_consume_loop(tmq, topic_list); + /*sync_consume_loop(tmq, topic_list);*/ } diff --git a/include/client/taos.h b/include/client/taos.h index 7c5cc56b20..a8627a43da 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -213,7 +213,7 @@ typedef struct tmq_message_t tmq_message_t; typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); DLL_EXPORT tmq_list_t *tmq_list_new(); -DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, char *); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); diff --git a/include/os/osEndian.h b/include/os/osEndian.h index e573ba0a75..1f52ece535 100644 --- a/include/os/osEndian.h +++ b/include/os/osEndian.h @@ -30,4 +30,4 @@ static const int32_t endian_test_var = 1; } #endif -#endif /*_TD_OS_ENDIAN_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENDIAN_H_*/ diff --git a/include/os/osMath.h b/include/os/osMath.h index 948fbbf665..3fe46d557e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -36,25 +36,25 @@ extern "C" { #else - #define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ + #define TSWAP(a, b, c) \ + do { \ + __typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ } while (0) - #define TMAX(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a > __b) ? __a : __b; \ + #define TMAX(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a > __b) ? __a : __b; \ }) - #define TMIN(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ + #define TMIN(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a < __b) ? __a : __b; \ }) #endif diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index facc8c7216..e57901ed2e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -146,7 +148,7 @@ tmq_list_t* tmq_list_new() { return ptr; } -int32_t tmq_list_append(tmq_list_t* ptr, char* src) { +int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { if (ptr->cnt >= ptr->tot - 1) return -1; ptr->elems[ptr->cnt] = strdup(src); ptr->cnt++; @@ -366,7 +368,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i .igExists = 1, .physicalPlan = (char*)pStr, .sql = (char*)sql, - .logicalPlan = "no logic plan", + .logicalPlan = (char*)"no logic plan", }; int tlen = tSerializeSCMCreateTopicReq(NULL, &req); @@ -512,7 +514,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return -1; } tDecodeSMqConsumeRsp(pMsg->pData, pRsp); - /*printf("rsp %ld %ld\n", pRsp->committedOffset, pRsp->rspOffset);*/ + /*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->numOfTopics == 0) { /*printf("no data\n");*/ free(pRsp); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 73f755bf1a..1cab413c3f 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -630,14 +630,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -#define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - - void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 6fd61baee3..fd960eb63e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -54,13 +55,14 @@ void mndCleanupConsumer(SMnode *pMnode) {} SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -88,6 +90,7 @@ CM_ENCODE_OVER: SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; @@ -106,7 +109,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - void *buf = malloc(len); + buf = malloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index bf138d335b..938a3eb89e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndSubscribe.h" #include "mndConsumer.h" @@ -372,13 +373,14 @@ void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -406,6 +408,7 @@ SUB_ENCODE_OVER: static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; @@ -425,7 +428,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - void *buf = malloc(tlen + 1); + buf = malloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index a9ba825a29..cefe9eff72 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -16,9 +16,9 @@ #ifndef _TD_TQ_INT_H_ #define _TD_TQ_INT_H_ -#include "tq.h" #include "meta.h" #include "tlog.h" +#include "tq.h" #include "trpc.h" #ifdef __cplusplus extern "C" { @@ -26,29 +26,48 @@ extern "C" { extern int32_t tqDebugFlag; -#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} -#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} -#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} -#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} -#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} -#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqFatal(...) \ + { \ + if (tqDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define tqError(...) \ + { \ + if (tqDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define tqWarn(...) \ + { \ + if (tqDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ + } \ + } +#define tqInfo(...) \ + { \ + if (tqDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TQ ", 255, __VA_ARGS__); \ + } \ + } +#define tqDebug(...) \ + { \ + if (tqDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } +#define tqTrace(...) \ + { \ + if (tqDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } -// create persistent storage for meta info such as consuming offset -// return value > 0: cgId -// return value <= 0: error code -// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); -// create ring buffer in memory and load consuming offset -// int tqOpenTCGroup(STQ*, const char* topic, int cgId); -// destroy ring buffer and persist consuming offset -// int tqCloseTCGroup(STQ*, const char* topic, int cgId); -// delete persistent storage for meta info -// int tqDropTCGroup(STQ*, const char* topic, int cgId); +int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); -//int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -//const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4e52ecd858..a625980505 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -12,19 +12,12 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -// static -// read next version data -// -// send to fetch queue -// -// handle management message -// - int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); if (old == 1) return 0; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 41fff4d228..1d9201f69d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -170,7 +170,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } if (pRead->pHead->head.version != ver) { - /*wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver);*/ + wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -178,7 +178,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { - /*wError("unexpected wal log version: checksum not passed");*/ + wError("unexpected wal log version: checksum not passed"); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 04efbef5b3..8999646f6a 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "os.h" #include "osString.h" diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index ddd283ae0f..0456d6a2ee 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "tbuffer.h" #include "exception.h" #include "os.h" diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 5ce69f1c2e..12aa77214f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,7 +19,6 @@ #include "tnote.h" #include "tutil.h" #include "ulog.h" -//#include "zlib.h" #define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) diff --git a/source/util/src/tpagedfile.c b/source/util/src/tpagedfile.c index 3373d09876..3cdba580d4 100644 --- a/source/util/src/tpagedfile.c +++ b/source/util/src/tpagedfile.c @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + #include "tpagedfile.h" #include "thash.h" #include "stddef.h" From 70644256323e69ae7c5819c83c57d2233e6f19ac Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 4 Feb 2022 21:10:36 +0800 Subject: [PATCH 42/42] fix compile --- source/util/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 6d8dfa51e7..760a66a4fb 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -10,6 +10,7 @@ target_link_libraries( util PRIVATE os PUBLIC lz4_static + PUBLIC api ) if(${BUILD_TEST})