From 1c27c2123fcb2e88114709c0e55bfaab5084c9d9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 20 Sep 2023 11:06:59 +0800 Subject: [PATCH 001/177] enhance: select `tbname` from (select tbname from d.st) --- include/libs/nodes/querynodes.h | 1 + source/libs/parser/inc/parAst.h | 1 + source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/parAstCreater.c | 15 ++ source/libs/parser/src/parTranslater.c | 3 +- source/libs/parser/src/sql.c | 233 +++++++++++++------------ 6 files changed, 138 insertions(+), 117 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 972421b1e7..58cc3c4a75 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -35,6 +35,7 @@ typedef struct SRawExprNode { char* p; uint32_t n; SNode* pNode; + bool isPseudoColumn; } SRawExprNode; typedef struct SDataType { diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 47043fddb1..3024eea397 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -98,6 +98,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode); SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode); +SNode* setRawExprNodeIsPseudoColumn(SAstCreateContext* pCxt, SNode* pNode, bool isPseudoColumn); SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode); SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 667b21893e..c7da1e0878 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -809,7 +809,7 @@ expr_or_subquery(A) ::= expression(B). //expr_or_subquery(A) ::= subquery(B). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), NULL); } expression(A) ::= literal(B). { A = B; } -expression(A) ::= pseudo_column(B). { A = B; } +expression(A) ::= pseudo_column(B). { A = B; setRawExprNodeIsPseudoColumn(pCxt, A, true); } expression(A) ::= column_reference(B). { A = B; } expression(A) ::= function_expression(B). { A = B; } expression(A) ::= case_when_expression(B). { A = B; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index e0b135b4f9..9baacf2790 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -257,6 +257,15 @@ SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const return (SNode*)target; } +SNode* setRawExprNodeIsPseudoColumn(SAstCreateContext* pCxt, SNode* pNode, bool isPseudoColumn) { + CHECK_PARSER_STATUS(pCxt); + if (NULL == pNode || QUERY_NODE_RAW_EXPR != nodeType(pNode)) { + return pNode; + } + ((SRawExprNode*)pNode)->isPseudoColumn = isPseudoColumn; + return pNode; +} + SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { CHECK_PARSER_STATUS(pCxt); SRawExprNode* pRawExpr = (SRawExprNode*)pNode; @@ -266,6 +275,12 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { if (QUERY_NODE_COLUMN == nodeType(pExpr)) { strcpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName); strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); + } else if (pRawExpr->isPseudoColumn) { + int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); + strncpy(pExpr->userAlias, pRawExpr->p, len); + pExpr->userAlias[len] = '\0'; + strncpy(pExpr->aliasName, pRawExpr->p, len); + pExpr->aliasName[len] = '\0'; } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c702400526..0fc57d3729 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1576,8 +1576,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN return TSDB_CODE_SUCCESS; } if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable || - QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } else { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ff1d834791..b86edc8791 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -6814,7 +6814,6 @@ static YYACTIONTYPE yy_reduce( case 405: /* signed_literal ::= signed */ yytestcase(yyruleno==405); case 426: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==426); case 427: /* expression ::= literal */ yytestcase(yyruleno==427); - case 428: /* expression ::= pseudo_column */ yytestcase(yyruleno==428); case 429: /* expression ::= column_reference */ yytestcase(yyruleno==429); case 430: /* expression ::= function_expression */ yytestcase(yyruleno==430); case 431: /* expression ::= case_when_expression */ yytestcase(yyruleno==431); @@ -6833,37 +6832,37 @@ static YYACTIONTYPE yy_reduce( case 597: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==597); #line 727 "sql.y" { yylhsminor.yy122 = yymsp[0].minor.yy122; } -#line 6836 "sql.c" +#line 6835 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 396: /* literal ::= NULL */ #line 728 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } -#line 6842 "sql.c" +#line 6841 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 397: /* literal ::= NK_QUESTION */ #line 729 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6848 "sql.c" +#line 6847 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 398: /* duration_literal ::= NK_VARIABLE */ #line 731 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6854 "sql.c" +#line 6853 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 399: /* signed ::= NK_INTEGER */ #line 733 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6860 "sql.c" +#line 6859 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 400: /* signed ::= NK_PLUS NK_INTEGER */ #line 734 "sql.y" { yymsp[-1].minor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6866 "sql.c" +#line 6865 "sql.c" break; case 401: /* signed ::= NK_MINUS NK_INTEGER */ #line 735 "sql.y" @@ -6872,19 +6871,19 @@ static YYACTIONTYPE yy_reduce( t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -#line 6875 "sql.c" +#line 6874 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 402: /* signed ::= NK_FLOAT */ #line 740 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6881 "sql.c" +#line 6880 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 403: /* signed ::= NK_PLUS NK_FLOAT */ #line 741 "sql.y" { yymsp[-1].minor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6887 "sql.c" +#line 6886 "sql.c" break; case 404: /* signed ::= NK_MINUS NK_FLOAT */ #line 742 "sql.y" @@ -6893,25 +6892,25 @@ static YYACTIONTYPE yy_reduce( t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } -#line 6896 "sql.c" +#line 6895 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 406: /* signed_literal ::= NK_STRING */ #line 749 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } -#line 6902 "sql.c" +#line 6901 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 407: /* signed_literal ::= NK_BOOL */ #line 750 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } -#line 6908 "sql.c" +#line 6907 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 408: /* signed_literal ::= TIMESTAMP NK_STRING */ #line 751 "sql.y" { yymsp[-1].minor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 6914 "sql.c" +#line 6913 "sql.c" break; case 409: /* signed_literal ::= duration_literal */ case 411: /* signed_literal ::= literal_func */ yytestcase(yyruleno==411); @@ -6923,19 +6922,25 @@ static YYACTIONTYPE yy_reduce( case 611: /* search_condition ::= common_expression */ yytestcase(yyruleno==611); #line 752 "sql.y" { yylhsminor.yy122 = releaseRawExprNode(pCxt, yymsp[0].minor.yy122); } -#line 6926 "sql.c" +#line 6925 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 410: /* signed_literal ::= NULL */ #line 753 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } -#line 6932 "sql.c" +#line 6931 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 412: /* signed_literal ::= NK_QUESTION */ #line 755 "sql.y" { yylhsminor.yy122 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } -#line 6938 "sql.c" +#line 6937 "sql.c" + yymsp[0].minor.yy122 = yylhsminor.yy122; + break; + case 428: /* expression ::= pseudo_column */ +#line 812 "sql.y" +{ yylhsminor.yy122 = yymsp[0].minor.yy122; setRawExprNodeIsPseudoColumn(pCxt, yylhsminor.yy122, true); } +#line 6943 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 432: /* expression ::= NK_LP expression NK_RP */ @@ -6943,7 +6948,7 @@ static YYACTIONTYPE yy_reduce( case 610: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==610); #line 816 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 6946 "sql.c" +#line 6951 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 433: /* expression ::= NK_PLUS expr_or_subquery */ @@ -6952,7 +6957,7 @@ static YYACTIONTYPE yy_reduce( SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy122)); } -#line 6955 "sql.c" +#line 6960 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 434: /* expression ::= NK_MINUS expr_or_subquery */ @@ -6961,7 +6966,7 @@ static YYACTIONTYPE yy_reduce( SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy122), NULL)); } -#line 6964 "sql.c" +#line 6969 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 435: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ @@ -6971,7 +6976,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 6974 "sql.c" +#line 6979 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 436: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ @@ -6981,7 +6986,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 6984 "sql.c" +#line 6989 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 437: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ @@ -6991,7 +6996,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 6994 "sql.c" +#line 6999 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 438: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ @@ -7001,7 +7006,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7004 "sql.c" +#line 7009 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 439: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ @@ -7011,7 +7016,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7014 "sql.c" +#line 7019 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 440: /* expression ::= column_reference NK_ARROW NK_STRING */ @@ -7020,7 +7025,7 @@ static YYACTIONTYPE yy_reduce( SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } -#line 7023 "sql.c" +#line 7028 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 441: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ @@ -7030,7 +7035,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7033 "sql.c" +#line 7038 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 442: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ @@ -7040,19 +7045,19 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7043 "sql.c" +#line 7048 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 445: /* column_reference ::= column_name */ #line 870 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy203, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy203)); } -#line 7049 "sql.c" +#line 7054 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 446: /* column_reference ::= table_name NK_DOT column_name */ #line 871 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy203, createColumnNode(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy203)); } -#line 7055 "sql.c" +#line 7060 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 447: /* pseudo_column ::= ROWTS */ @@ -7069,68 +7074,68 @@ static YYACTIONTYPE yy_reduce( case 464: /* literal_func ::= NOW */ yytestcase(yyruleno==464); #line 873 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } -#line 7072 "sql.c" +#line 7077 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 449: /* pseudo_column ::= table_name NK_DOT TBNAME */ #line 875 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy203)))); } -#line 7078 "sql.c" +#line 7083 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 459: /* function_expression ::= function_name NK_LP expression_list NK_RP */ case 460: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==460); #line 886 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy203, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy203, yymsp[-1].minor.yy298)); } -#line 7085 "sql.c" +#line 7090 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 461: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ #line 889 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), yymsp[-1].minor.yy388)); } -#line 7091 "sql.c" +#line 7096 "sql.c" yymsp[-5].minor.yy122 = yylhsminor.yy122; break; case 463: /* literal_func ::= noarg_func NK_LP NK_RP */ #line 892 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy203, NULL)); } -#line 7097 "sql.c" +#line 7102 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 478: /* star_func_para_list ::= NK_STAR */ #line 916 "sql.y" { yylhsminor.yy298 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } -#line 7103 "sql.c" +#line 7108 "sql.c" yymsp[0].minor.yy298 = yylhsminor.yy298; break; case 483: /* star_func_para ::= table_name NK_DOT NK_STAR */ case 551: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==551); #line 925 "sql.y" { yylhsminor.yy122 = createColumnNode(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy0); } -#line 7110 "sql.c" +#line 7115 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 484: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ #line 928 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy298, yymsp[-1].minor.yy122)); } -#line 7116 "sql.c" +#line 7121 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 485: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ #line 930 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), yymsp[-2].minor.yy298, yymsp[-1].minor.yy122)); } -#line 7122 "sql.c" +#line 7127 "sql.c" yymsp[-4].minor.yy122 = yylhsminor.yy122; break; case 488: /* when_then_expr ::= WHEN common_expression THEN common_expression */ #line 937 "sql.y" { yymsp[-3].minor.yy122 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122)); } -#line 7128 "sql.c" +#line 7133 "sql.c" break; case 490: /* case_when_else_opt ::= ELSE common_expression */ #line 940 "sql.y" { yymsp[-1].minor.yy122 = releaseRawExprNode(pCxt, yymsp[0].minor.yy122); } -#line 7133 "sql.c" +#line 7138 "sql.c" break; case 491: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ case 496: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==496); @@ -7140,7 +7145,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy416, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7143 "sql.c" +#line 7148 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 492: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ @@ -7150,7 +7155,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy122), releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7153 "sql.c" +#line 7158 "sql.c" yymsp[-4].minor.yy122 = yylhsminor.yy122; break; case 493: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ @@ -7160,7 +7165,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy122), releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7163 "sql.c" +#line 7168 "sql.c" yymsp[-5].minor.yy122 = yylhsminor.yy122; break; case 494: /* predicate ::= expr_or_subquery IS NULL */ @@ -7169,7 +7174,7 @@ static YYACTIONTYPE yy_reduce( SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), NULL)); } -#line 7172 "sql.c" +#line 7177 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 495: /* predicate ::= expr_or_subquery IS NOT NULL */ @@ -7178,78 +7183,78 @@ static YYACTIONTYPE yy_reduce( SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), NULL)); } -#line 7181 "sql.c" +#line 7186 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 497: /* compare_op ::= NK_LT */ #line 977 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_LOWER_THAN; } -#line 7187 "sql.c" +#line 7192 "sql.c" break; case 498: /* compare_op ::= NK_GT */ #line 978 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_GREATER_THAN; } -#line 7192 "sql.c" +#line 7197 "sql.c" break; case 499: /* compare_op ::= NK_LE */ #line 979 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_LOWER_EQUAL; } -#line 7197 "sql.c" +#line 7202 "sql.c" break; case 500: /* compare_op ::= NK_GE */ #line 980 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_GREATER_EQUAL; } -#line 7202 "sql.c" +#line 7207 "sql.c" break; case 501: /* compare_op ::= NK_NE */ #line 981 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_NOT_EQUAL; } -#line 7207 "sql.c" +#line 7212 "sql.c" break; case 502: /* compare_op ::= NK_EQ */ #line 982 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_EQUAL; } -#line 7212 "sql.c" +#line 7217 "sql.c" break; case 503: /* compare_op ::= LIKE */ #line 983 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_LIKE; } -#line 7217 "sql.c" +#line 7222 "sql.c" break; case 504: /* compare_op ::= NOT LIKE */ #line 984 "sql.y" { yymsp[-1].minor.yy416 = OP_TYPE_NOT_LIKE; } -#line 7222 "sql.c" +#line 7227 "sql.c" break; case 505: /* compare_op ::= MATCH */ #line 985 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_MATCH; } -#line 7227 "sql.c" +#line 7232 "sql.c" break; case 506: /* compare_op ::= NMATCH */ #line 986 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_NMATCH; } -#line 7232 "sql.c" +#line 7237 "sql.c" break; case 507: /* compare_op ::= CONTAINS */ #line 987 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_JSON_CONTAINS; } -#line 7237 "sql.c" +#line 7242 "sql.c" break; case 508: /* in_op ::= IN */ #line 991 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_IN; } -#line 7242 "sql.c" +#line 7247 "sql.c" break; case 509: /* in_op ::= NOT IN */ #line 992 "sql.y" { yymsp[-1].minor.yy416 = OP_TYPE_NOT_IN; } -#line 7247 "sql.c" +#line 7252 "sql.c" break; case 510: /* in_predicate_value ::= NK_LP literal_list NK_RP */ #line 994 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy298)); } -#line 7252 "sql.c" +#line 7257 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 512: /* boolean_value_expression ::= NOT boolean_primary */ @@ -7258,7 +7263,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy122), NULL)); } -#line 7261 "sql.c" +#line 7266 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 513: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ @@ -7268,7 +7273,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7271 "sql.c" +#line 7276 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 514: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ @@ -7278,64 +7283,64 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7281 "sql.c" +#line 7286 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 522: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ #line 1027 "sql.y" { yylhsminor.yy122 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy122, yymsp[0].minor.yy122, NULL); } -#line 7287 "sql.c" +#line 7292 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 525: /* table_primary ::= table_name alias_opt */ #line 1033 "sql.y" { yylhsminor.yy122 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy203, &yymsp[0].minor.yy203); } -#line 7293 "sql.c" +#line 7298 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 526: /* table_primary ::= db_name NK_DOT table_name alias_opt */ #line 1034 "sql.y" { yylhsminor.yy122 = createRealTableNode(pCxt, &yymsp[-3].minor.yy203, &yymsp[-1].minor.yy203, &yymsp[0].minor.yy203); } -#line 7299 "sql.c" +#line 7304 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 527: /* table_primary ::= subquery alias_opt */ #line 1035 "sql.y" { yylhsminor.yy122 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122), &yymsp[0].minor.yy203); } -#line 7305 "sql.c" +#line 7310 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 529: /* alias_opt ::= */ #line 1040 "sql.y" { yymsp[1].minor.yy203 = nil_token; } -#line 7311 "sql.c" +#line 7316 "sql.c" break; case 531: /* alias_opt ::= AS table_alias */ #line 1042 "sql.y" { yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; } -#line 7316 "sql.c" +#line 7321 "sql.c" break; case 532: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ case 533: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==533); #line 1044 "sql.y" { yymsp[-2].minor.yy122 = yymsp[-1].minor.yy122; } -#line 7322 "sql.c" +#line 7327 "sql.c" break; case 534: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ #line 1049 "sql.y" { yylhsminor.yy122 = createJoinTableNode(pCxt, yymsp[-4].minor.yy498, yymsp[-5].minor.yy122, yymsp[-2].minor.yy122, yymsp[0].minor.yy122); } -#line 7327 "sql.c" +#line 7332 "sql.c" yymsp[-5].minor.yy122 = yylhsminor.yy122; break; case 535: /* join_type ::= */ #line 1053 "sql.y" { yymsp[1].minor.yy498 = JOIN_TYPE_INNER; } -#line 7333 "sql.c" +#line 7338 "sql.c" break; case 536: /* join_type ::= INNER */ #line 1054 "sql.y" { yymsp[0].minor.yy498 = JOIN_TYPE_INNER; } -#line 7338 "sql.c" +#line 7343 "sql.c" break; case 537: /* query_specification ::= SELECT hint_list set_quantifier_opt tag_mode_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ #line 1060 "sql.y" @@ -7351,42 +7356,42 @@ static YYACTIONTYPE yy_reduce( yymsp[-13].minor.yy122 = addEveryClause(pCxt, yymsp[-13].minor.yy122, yymsp[-4].minor.yy122); yymsp[-13].minor.yy122 = addFillClause(pCxt, yymsp[-13].minor.yy122, yymsp[-3].minor.yy122); } -#line 7354 "sql.c" +#line 7359 "sql.c" break; case 538: /* hint_list ::= */ #line 1075 "sql.y" { yymsp[1].minor.yy298 = createHintNodeList(pCxt, NULL); } -#line 7359 "sql.c" +#line 7364 "sql.c" break; case 539: /* hint_list ::= NK_HINT */ #line 1076 "sql.y" { yylhsminor.yy298 = createHintNodeList(pCxt, &yymsp[0].minor.yy0); } -#line 7364 "sql.c" +#line 7369 "sql.c" yymsp[0].minor.yy298 = yylhsminor.yy298; break; case 544: /* set_quantifier_opt ::= ALL */ #line 1087 "sql.y" { yymsp[0].minor.yy983 = false; } -#line 7370 "sql.c" +#line 7375 "sql.c" break; case 547: /* select_item ::= NK_STAR */ #line 1094 "sql.y" { yylhsminor.yy122 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } -#line 7375 "sql.c" +#line 7380 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 549: /* select_item ::= common_expression column_alias */ case 559: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==559); #line 1096 "sql.y" { yylhsminor.yy122 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122), &yymsp[0].minor.yy203); } -#line 7382 "sql.c" +#line 7387 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 550: /* select_item ::= common_expression AS column_alias */ case 560: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==560); #line 1097 "sql.y" { yylhsminor.yy122 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), &yymsp[0].minor.yy203); } -#line 7389 "sql.c" +#line 7394 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 555: /* partition_by_clause_opt ::= PARTITION BY partition_list */ @@ -7394,99 +7399,99 @@ static YYACTIONTYPE yy_reduce( case 600: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==600); #line 1106 "sql.y" { yymsp[-2].minor.yy298 = yymsp[0].minor.yy298; } -#line 7397 "sql.c" +#line 7402 "sql.c" break; case 562: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ #line 1119 "sql.y" { yymsp[-5].minor.yy122 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7402 "sql.c" +#line 7407 "sql.c" break; case 563: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ #line 1120 "sql.y" { yymsp[-3].minor.yy122 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7407 "sql.c" +#line 7412 "sql.c" break; case 564: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ #line 1122 "sql.y" { yymsp[-5].minor.yy122 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), NULL, yymsp[-1].minor.yy122, yymsp[0].minor.yy122); } -#line 7412 "sql.c" +#line 7417 "sql.c" break; case 565: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ #line 1125 "sql.y" { yymsp[-7].minor.yy122 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy122), releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), yymsp[-1].minor.yy122, yymsp[0].minor.yy122); } -#line 7417 "sql.c" +#line 7422 "sql.c" break; case 566: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ #line 1127 "sql.y" { yymsp[-6].minor.yy122 = createEventWindowNode(pCxt, yymsp[-3].minor.yy122, yymsp[0].minor.yy122); } -#line 7422 "sql.c" +#line 7427 "sql.c" break; case 570: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ #line 1133 "sql.y" { yymsp[-3].minor.yy122 = createFillNode(pCxt, yymsp[-1].minor.yy312, NULL); } -#line 7427 "sql.c" +#line 7432 "sql.c" break; case 571: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ #line 1134 "sql.y" { yymsp[-5].minor.yy122 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy298)); } -#line 7432 "sql.c" +#line 7437 "sql.c" break; case 572: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ #line 1135 "sql.y" { yymsp[-5].minor.yy122 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy298)); } -#line 7437 "sql.c" +#line 7442 "sql.c" break; case 573: /* fill_mode ::= NONE */ #line 1139 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NONE; } -#line 7442 "sql.c" +#line 7447 "sql.c" break; case 574: /* fill_mode ::= PREV */ #line 1140 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_PREV; } -#line 7447 "sql.c" +#line 7452 "sql.c" break; case 575: /* fill_mode ::= NULL */ #line 1141 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NULL; } -#line 7452 "sql.c" +#line 7457 "sql.c" break; case 576: /* fill_mode ::= NULL_F */ #line 1142 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NULL_F; } -#line 7457 "sql.c" +#line 7462 "sql.c" break; case 577: /* fill_mode ::= LINEAR */ #line 1143 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_LINEAR; } -#line 7462 "sql.c" +#line 7467 "sql.c" break; case 578: /* fill_mode ::= NEXT */ #line 1144 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NEXT; } -#line 7467 "sql.c" +#line 7472 "sql.c" break; case 581: /* group_by_list ::= expr_or_subquery */ #line 1153 "sql.y" { yylhsminor.yy298 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7472 "sql.c" +#line 7477 "sql.c" yymsp[0].minor.yy298 = yylhsminor.yy298; break; case 582: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ #line 1154 "sql.y" { yylhsminor.yy298 = addNodeToList(pCxt, yymsp[-2].minor.yy298, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7478 "sql.c" +#line 7483 "sql.c" yymsp[-2].minor.yy298 = yylhsminor.yy298; break; case 586: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ #line 1161 "sql.y" { yymsp[-5].minor.yy122 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7484 "sql.c" +#line 7489 "sql.c" break; case 587: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ #line 1163 "sql.y" { yymsp[-3].minor.yy122 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7489 "sql.c" +#line 7494 "sql.c" break; case 590: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ #line 1170 "sql.y" @@ -7495,80 +7500,80 @@ static YYACTIONTYPE yy_reduce( yylhsminor.yy122 = addSlimitClause(pCxt, yylhsminor.yy122, yymsp[-1].minor.yy122); yylhsminor.yy122 = addLimitClause(pCxt, yylhsminor.yy122, yymsp[0].minor.yy122); } -#line 7498 "sql.c" +#line 7503 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 593: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ #line 1180 "sql.y" { yylhsminor.yy122 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy122, yymsp[0].minor.yy122); } -#line 7504 "sql.c" +#line 7509 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 594: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ #line 1182 "sql.y" { yylhsminor.yy122 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy122, yymsp[0].minor.yy122); } -#line 7510 "sql.c" +#line 7515 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 602: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ case 606: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==606); #line 1196 "sql.y" { yymsp[-1].minor.yy122 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 7517 "sql.c" +#line 7522 "sql.c" break; case 603: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ case 607: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==607); #line 1197 "sql.y" { yymsp[-3].minor.yy122 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } -#line 7523 "sql.c" +#line 7528 "sql.c" break; case 604: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ case 608: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==608); #line 1198 "sql.y" { yymsp[-3].minor.yy122 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } -#line 7529 "sql.c" +#line 7534 "sql.c" break; case 609: /* subquery ::= NK_LP query_expression NK_RP */ #line 1206 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy122); } -#line 7534 "sql.c" +#line 7539 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 614: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ #line 1220 "sql.y" { yylhsminor.yy122 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), yymsp[-1].minor.yy626, yymsp[0].minor.yy877); } -#line 7540 "sql.c" +#line 7545 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 615: /* ordering_specification_opt ::= */ #line 1224 "sql.y" { yymsp[1].minor.yy626 = ORDER_ASC; } -#line 7546 "sql.c" +#line 7551 "sql.c" break; case 616: /* ordering_specification_opt ::= ASC */ #line 1225 "sql.y" { yymsp[0].minor.yy626 = ORDER_ASC; } -#line 7551 "sql.c" +#line 7556 "sql.c" break; case 617: /* ordering_specification_opt ::= DESC */ #line 1226 "sql.y" { yymsp[0].minor.yy626 = ORDER_DESC; } -#line 7556 "sql.c" +#line 7561 "sql.c" break; case 618: /* null_ordering_opt ::= */ #line 1230 "sql.y" { yymsp[1].minor.yy877 = NULL_ORDER_DEFAULT; } -#line 7561 "sql.c" +#line 7566 "sql.c" break; case 619: /* null_ordering_opt ::= NULLS FIRST */ #line 1231 "sql.y" { yymsp[-1].minor.yy877 = NULL_ORDER_FIRST; } -#line 7566 "sql.c" +#line 7571 "sql.c" break; case 620: /* null_ordering_opt ::= NULLS LAST */ #line 1232 "sql.y" { yymsp[-1].minor.yy877 = NULL_ORDER_LAST; } -#line 7571 "sql.c" +#line 7576 "sql.c" break; default: break; @@ -7641,7 +7646,7 @@ static void yy_syntax_error( } else if (TSDB_CODE_PAR_DB_NOT_SPECIFIED == pCxt->errCode && TK_NK_FLOAT == TOKEN.type) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } -#line 7644 "sql.c" +#line 7649 "sql.c" /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE From 29458b55e50301b4e077e4b0969af2d2df9119f1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 25 Sep 2023 17:16:19 +0800 Subject: [PATCH 002/177] test: increase numbers of cases in windows --- tests/system-test/simpletest.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat index 5ae2d3feb3..31b76cad4a 100644 --- a/tests/system-test/simpletest.bat +++ b/tests/system-test/simpletest.bat @@ -18,7 +18,7 @@ python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py @REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py @REM python3 .\test.py -f 1-insert\alter_stable.py @REM python3 .\test.py -f 1-insert\alter_table.py -@REM python3 .\test.py -f 2-query\between.py +python3 .\test.py -f 2-query\between.py @REM python3 .\test.py -f 2-query\distinct.py @REM python3 .\test.py -f 2-query\varchar.py @REM python3 .\test.py -f 2-query\ltrim.py @@ -101,3 +101,4 @@ python3 .\test.py -f 7-tmq\subscribeStb.py @REM python3 .\test.py -f 7-tmq\subscribeStb3.py @REM python3 .\test.py -f 7-tmq\subscribeStb4.py @REM python3 .\test.py -f 7-tmq\db.py +python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file From b1014f31bca665cf17bddc1881fb8c4e2f5b3e5a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 26 Sep 2023 20:45:40 +0800 Subject: [PATCH 003/177] test: add testcase of TS-4074 --- tests/system-test/0-others/compatibility.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index cb804aad0c..83bfb2bed7 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -30,7 +30,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -38,7 +38,9 @@ class TDTestCase: delete from deldata.stb1; flush database deldata; insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); - delete from deldata.ct1;''' + delete from deldata.ct1; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ); + flush database deldata;''' def checkProcessPid(self,processName): i=0 while i<60: @@ -262,7 +264,7 @@ class TDTestCase: if self.is_list_same_as_ordered_list(resultList,expectList): print("The unordered list is the same as the ordered list.") else: - tdlog.error("The unordered list is not the same as the ordered list.") + tdLog.exit("The unordered list is not the same as the ordered list.") tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") From 65448e20269039b4d2ab165d09e7c8c6c9516f79 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 27 Sep 2023 10:56:54 +0800 Subject: [PATCH 004/177] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 2 + tests/system-test/7-tmq/tmqVnodeSplit.py | 1 - tests/system-test/7-tmq/tmqVnodeTransform.py | 44 ++++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9995c2970f..cf161ab004 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -168,7 +168,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py diff --git a/tests/system-test/7-tmq/tmqVnodeSplit.py b/tests/system-test/7-tmq/tmqVnodeSplit.py index c6cdc2bf83..67deb84620 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit.py @@ -194,7 +194,6 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() self.prepareTestEnv() self.tmqCase1(True) self.prepareTestEnv() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index fa50e46853..3ebf48b93a 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -58,7 +58,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdCom.drop_all_db(); + tdCom.drop_all_db() tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") @@ -102,7 +102,7 @@ class TDTestCase: tdSql.query("show dnodes") for result in tdSql.queryResult: dnodesList.append(result[0]) - + print("dnodeList:",dnodesList) tdSql.query("select * from information_schema.ins_vnodes") vnodeId = 0 for result in tdSql.queryResult: @@ -110,9 +110,16 @@ class TDTestCase: tdLog.debug("dnode is %d"%(result[0])) dnodesList.remove(result[0]) vnodeId = result[1] - break - redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) - tdLog.debug("redistributeSql:%s"%(redistributeSql)) + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") @@ -179,7 +186,7 @@ class TDTestCase: tmqCom.getStartCommitNotifyFromTmqsim() #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup self.redistributeVgroups(); @@ -228,7 +235,7 @@ class TDTestCase: tdSql.execute(sqlString) tdSql.query("flush database %s"%(paraDict['dbName'])) #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup self.redistributeVgroups(); @@ -236,7 +243,8 @@ class TDTestCase: sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) tdLog.info("alter table sql: %s"%sqlString) tdSql.error(sqlString) - + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) time.sleep(1) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) @@ -284,7 +292,7 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -305,27 +313,27 @@ class TDTestCase: time.sleep(5) #restart dnode & remove wal - self.restartAndRemoveWal() + # self.restartAndRemoveWal() # redistribute vgroup - self.redistributeVgroups(); + self.redistributeVgroups() tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - - time.sleep(10) + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(20) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) tdLog.printNoPrefix("======== test case 3 end ...... ") def run(self): - - tdSql.prepare() - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() + # self.prepareTestEnv() + # self.tmqCase1() + # self.tmqCase2() self.prepareTestEnv() self.tmqCase3() From dce29b367b9ad35e6cc350dd0f49a25ddaf610d4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 27 Sep 2023 13:46:14 +0800 Subject: [PATCH 005/177] fix: retore translater.c --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 81da234b06..959cd81e06 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1555,7 +1555,8 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN return TSDB_CODE_SUCCESS; } if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable || + QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } else { From 1aad905558e7118df00d5f884ef013fe7489ebdb Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 8 Oct 2023 15:55:01 +0800 Subject: [PATCH 006/177] fix: collection functions hash set distinct error --- source/libs/nodes/src/nodesUtilFuncs.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index bf5b6c8080..c5a1bfa599 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2033,7 +2033,6 @@ typedef struct SCollectFuncsCxt { char* tableAlias; FFuncClassifier classifier; SNodeList* pFuncs; - SHashObj* pFuncsSet; } SCollectFuncsCxt; static EDealRes collectFuncs(SNode* pNode, void* pContext) { @@ -2048,9 +2047,15 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { } } SExprNode* pExpr = (SExprNode*)pNode; - if (NULL == taosHashGet(pCxt->pFuncsSet, &pExpr, sizeof(SExprNode*))) { + bool bFound = false; + SNode* pn = NULL; + FOREACH(pn, pCxt->pFuncs) { + if (nodesEqualNode(pn, pNode)) { + bFound = true; + } + } + if (!bFound) { pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode)); - taosHashPut(pCxt->pFuncsSet, &pExpr, POINTER_BYTES, &pExpr, POINTER_BYTES); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } @@ -2077,12 +2082,10 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAl SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .tableAlias = tableAlias, - .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs), - .pFuncsSet = taosHashInit(4, funcNodeHash, false, false)}; - if (NULL == cxt.pFuncs || NULL == cxt.pFuncsSet) { + .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs)}; + if (NULL == cxt.pFuncs) { return TSDB_CODE_OUT_OF_MEMORY; } - taosHashSetEqualFp(cxt.pFuncsSet, funcNodeEqual); *pFuncs = NULL; nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS == cxt.errCode) { @@ -2094,7 +2097,6 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAl } else { nodesDestroyList(cxt.pFuncs); } - taosHashCleanup(cxt.pFuncsSet); return cxt.errCode; } From 3f2d8905607a63ded0599bbe08ccde03d6e49d56 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 8 Oct 2023 16:06:15 +0800 Subject: [PATCH 007/177] feat:[TD-26056] add replay logic --- include/common/tmsg.h | 90 ++------------------- include/libs/executor/storageapi.h | 1 + include/util/taoserror.h | 2 + source/client/src/clientSml.c | 2 +- source/client/src/clientSmlJson.c | 40 +++++----- source/client/src/clientSmlLine.c | 2 +- source/client/src/clientTmq.c | 57 ++++++++------ source/common/src/tmsg.c | 33 ++------ source/dnode/mnode/impl/inc/mndDef.h | 26 ------- source/dnode/mnode/impl/src/mndConsumer.c | 23 +++++- source/dnode/mnode/impl/src/mndDump.c | 20 +---- source/dnode/mnode/impl/src/mndTopic.c | 22 ------ source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/inc/tq.h | 11 +-- source/dnode/vnode/src/tq/tq.c | 3 + source/dnode/vnode/src/tq/tqOffset.c | 13 ++-- source/dnode/vnode/src/tq/tqRead.c | 8 +- source/dnode/vnode/src/tq/tqScan.c | 95 +++++++++++++++++------ source/dnode/vnode/src/tq/tqUtil.c | 4 +- source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + source/libs/executor/src/scanoperator.c | 3 + source/util/src/terror.c | 2 + utils/test/c/tmqOffset.c | 9 +-- 23 files changed, 192 insertions(+), 278 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index af44184de6..90147a3c31 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2295,17 +2295,6 @@ int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateS int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq); void tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq); -typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - int64_t streamId; - char* sql; - char* executorMsg; -} SMVCreateStreamReq, SMSCreateStreamReq; - -typedef struct { - int64_t streamId; -} SMVCreateStreamRsp, SMSCreateStreamRsp; - enum { TOPIC_SUB_TYPE__DB = 1, TOPIC_SUB_TYPE__TABLE, @@ -2327,16 +2316,9 @@ int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTo int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq); void tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq); -typedef struct { - int64_t topicId; -} SCMCreateTopicRsp; - -int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp); -int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp); - typedef struct { int64_t consumerId; -} SMqConsumerLostMsg, SMqConsumerRecoverMsg, SMqConsumerClearMsg; +} SMqConsumerRecoverMsg, SMqConsumerClearMsg; typedef struct { int64_t consumerId; @@ -2348,6 +2330,7 @@ typedef struct { int8_t autoCommit; int32_t autoCommitInterval; int8_t resetOffsetCfg; + int8_t enableReplay; } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -2367,6 +2350,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc tlen += taosEncodeFixedI8(buf, pReq->autoCommit); tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval); tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg); + tlen += taosEncodeFixedI8(buf, pReq->enableReplay); return tlen; } @@ -2390,71 +2374,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI8(buf, &pReq->autoCommit); buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval); buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); - return buf; -} - -typedef struct SMqSubTopic { - int32_t vgId; - int64_t topicId; - SEpSet epSet; -} SMqSubTopic; - -typedef struct { - int32_t topicNum; - SMqSubTopic topics[]; -} SCMSubscribeRsp; - -static FORCE_INLINE int32_t tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pRsp->topicNum); - for (int32_t i = 0; i < pRsp->topicNum; i++) { - tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId); - tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId); - tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet); - } - return tlen; -} - -static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { - buf = taosDecodeFixedI32(buf, &pRsp->topicNum); - for (int32_t i = 0; i < pRsp->topicNum; i++) { - buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId); - buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId); - buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet); - } - return buf; -} - -typedef struct { - int64_t topicId; - int64_t consumerId; - int64_t consumerGroupId; - int64_t offset; - char* sql; - char* logicalPlan; - char* physicalPlan; -} SMVSubscribeReq; - -static FORCE_INLINE int32_t tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->topicId); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId); - tlen += taosEncodeFixedI64(buf, pReq->offset); - tlen += taosEncodeString(buf, pReq->sql); - tlen += taosEncodeString(buf, pReq->logicalPlan); - tlen += taosEncodeString(buf, pReq->physicalPlan); - return tlen; -} - -static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->topicId); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeFixedI64(buf, &pReq->consumerGroupId); - buf = taosDecodeFixedI64(buf, &pReq->offset); - buf = taosDecodeString(buf, &pReq->sql); - buf = taosDecodeString(buf, &pReq->logicalPlan); - buf = taosDecodeString(buf, &pReq->physicalPlan); + buf = taosDecodeFixedI8(buf, &pReq->enableReplay); return buf; } @@ -3534,6 +3454,7 @@ typedef struct { int64_t consumerId; int64_t timeout; STqOffsetVal reqOffset; + int8_t enableReplay; } SMqPollReq; int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); @@ -3593,6 +3514,7 @@ typedef struct { SArray* blockData; SArray* blockTbName; SArray* blockSchema; + int64_t sleepTime; } SMqDataRsp; int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0a240dd8f5..fa110d66e2 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -222,6 +222,7 @@ typedef struct SStoreTqReader { bool (*tqReaderNextBlockInWal)(); bool (*tqNextBlockImpl)(); // todo remove it SSDataBlock* (*tqGetResultBlock)(); + int64_t (*tqGetResultBlockTime)(); void (*tqReaderSetColIdList)(); int32_t (*tqReaderSetQueryTableList)(); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 39bf2b5681..eb991379c4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -792,6 +792,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_NEED_INITIALIZED TAOS_DEF_ERROR_CODE(0, 0x4010) #define TSDB_CODE_TMQ_NO_COMMITTED TAOS_DEF_ERROR_CODE(0, 0x4011) #define TSDB_CODE_TMQ_SAME_COMMITTED_VALUE TAOS_DEF_ERROR_CODE(0, 0x4012) +#define TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP TAOS_DEF_ERROR_CODE(0, 0x4013) +#define TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x4014) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 10f8b89f4d..91c21fe344 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -683,7 +683,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { taosHashCleanup(hashTmp); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } } taosHashCleanup(hashTmp); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index f9076112c4..5e656c71a7 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -256,7 +256,8 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { } if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) return -1; + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); + return TSDB_CODE_TSC_INVALID_JSON; } char *sTmp = *start; @@ -367,7 +368,8 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || element->measure == NULL || element->timestamp == NULL) { - uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM) return -1; + uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM); + return TSDB_CODE_TSC_INVALID_JSON; } return 0; } @@ -381,7 +383,8 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) return -1; + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); + return TSDB_CODE_TSC_INVALID_JSON; } if ((*start)[1] == 'm') { @@ -448,7 +451,8 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { - uError("elements != %d", OTD_JSON_FIELDS_NUM) return -1; + uError("elements != %d", OTD_JSON_FIELDS_NUM); + return TSDB_CODE_TSC_INVALID_JSON; } return 0; } @@ -477,7 +481,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { } if (*marks[i] == NULL) { uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } } return TSDB_CODE_SUCCESS; @@ -816,25 +820,25 @@ static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPr int32_t size = cJSON_GetArraySize(root); if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cJSON *value = cJSON_GetObjectItem(root, "value"); if (unlikely(!cJSON_IsNumber(value))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cJSON *type = cJSON_GetObjectItem(root, "type"); if (unlikely(!cJSON_IsString(type))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } double timeDouble = value->valuedouble; if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } if (timeDouble == 0) { @@ -849,32 +853,28 @@ static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPr size_t typeLen = strlen(type->valuestring); if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { // seconds - int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) { return tsInt64 * smlFactorS[toPrecision]; } - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { switch (type->valuestring[0]) { case 'm': case 'M': // milliseconds return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); - break; case 'u': case 'U': // microseconds return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); - break; case 'n': case 'N': return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); - break; default: - return -1; + return TSDB_CODE_TSC_INVALID_JSON_TYPE; } } else { - return -1; + return TSDB_CODE_TSC_INVALID_JSON_TYPE; } } @@ -895,7 +895,7 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { double timeDouble = timestamp->valuedouble; if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } if (unlikely(timeDouble < 0)) { @@ -911,14 +911,14 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { if (unlikely(fromPrecision == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } int64_t tsInt64 = timeDouble; if (fromPrecision == TSDB_TIME_PRECISION_SECONDS) { if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) { return tsInt64 * smlFactorS[toPrecision]; } - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } else { return convertTimePrecision(timeDouble, fromPrecision, toPrecision); } @@ -926,7 +926,7 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { return smlParseTSFromJSONObj(info, timestamp, toPrecision); } else { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index a565fb1a21..006475654a 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -70,7 +70,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); if (unlikely(ts == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } return ts; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 6ee5508048..50b8eb1eca 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -63,7 +63,7 @@ struct tmq_conf_t { int8_t resetOffset; int8_t withTbName; int8_t snapEnable; - int32_t snapBatchSize; + int8_t replayEnable; bool hbBgEnable; uint16_t port; int32_t autoCommitInterval; @@ -83,6 +83,7 @@ struct tmq_t { int8_t autoCommit; int32_t autoCommitInterval; int8_t resetOffsetCfg; + int8_t replayEnable; uint64_t consumerId; bool hbBgEnable; tmq_commit_cb* commitCb; @@ -92,19 +93,13 @@ struct tmq_t { SRWLatch lock; int8_t status; int32_t epoch; -#if 0 - int8_t epStatus; - int32_t epSkipCnt; -#endif // poll info int64_t pollCnt; int64_t totalRows; -// bool needReportOffsetRows; // timer tmr_h hbLiveTimer; tmr_h epTimer; - tmr_h reportTimer; tmr_h commitTimer; STscObj* pTscObj; // connection SArray* clientTopics; // SArray @@ -152,6 +147,8 @@ typedef struct { int32_t vgStatus; int32_t vgSkipCnt; // here used to mark the slow vgroups int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int64_t blockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int64_t blockSleepForReplay; // once empty block is received, idle for ignoreCnt then start to poll data bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. SEpSet epSet; } SMqClientVg; @@ -360,24 +357,6 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } - if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) { - conf->snapBatchSize = taosStr2int64(value); - return TMQ_CONF_OK; - } - -// if (strcasecmp(key, "enable.heartbeat.background") == 0) { - // if (strcasecmp(value, "true") == 0) { - // conf->hbBgEnable = true; - // return TMQ_CONF_OK; - // } else if (strcasecmp(value, "false") == 0) { - // conf->hbBgEnable = false; - // return TMQ_CONF_OK; - // } else { -// tscError("the default value of enable.heartbeat.background is true, can not be seted"); -// return TMQ_CONF_INVALID; - // } -// } - if (strcasecmp(key, "td.connect.ip") == 0) { conf->ip = taosStrdup(value); return TMQ_CONF_OK; @@ -398,6 +377,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } + if (strcasecmp(key, "enable.replay") == 0) { + if (strcasecmp(value, "true") == 0) { + conf->replayEnable = true; + return TMQ_CONF_OK; + } else if (strcasecmp(value, "false") == 0) { + conf->replayEnable = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } + if (strcasecmp(key, "td.connect.db") == 0) { return TMQ_CONF_OK; } @@ -1075,6 +1066,10 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + pTmq->replayEnable = conf->replayEnable; + if(conf->replayEnable){ + pTmq->autoCommit = false; + } taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1424,6 +1419,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, .vgSkipCnt = 0, .emptyBlockReceiveTs = 0, + .blockReceiveTs = 0, + .blockSleepForReplay = 0, .numOfRows = pInfo ? pInfo->numOfRows : 0, }; @@ -1695,6 +1692,12 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { continue; } + if (tmq->replayEnable && taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) { // less than 10ms + tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId, + tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); + continue; + } + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); @@ -1816,6 +1819,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; + if(tmq->replayEnable){ + pVg->blockReceiveTs = taosGetTimestampMs(); + pVg->blockSleepForReplay = pRsp->rsp.sleepTime; + } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index fd39ae98d9..7d285434d9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4352,31 +4352,6 @@ void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { } } -int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->topicId) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->topicId) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -5983,6 +5958,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1; if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1; + if (tEncodeI8(&encoder, pReq->enableReplay) < 0) return -1; tEndEncode(&encoder); @@ -6019,6 +5995,10 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1; if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->enableReplay) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -7795,6 +7775,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { } } } + if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1; return 0; } @@ -7840,6 +7821,8 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } } } + if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + return 0; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 844e69e659..1715f56091 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -490,32 +490,6 @@ typedef struct { char filterTb[TSDB_TABLE_NAME_LEN]; } SShowObj; -typedef struct { - int64_t id; - int8_t type; - int8_t replica; - int16_t numOfColumns; - int32_t rowSize; - int32_t numOfRows; - int32_t numOfReads; - int32_t payloadLen; - void* pIter; - SMnode* pMnode; - char db[TSDB_DB_FNAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; - char payload[]; -} SSysTableRetrieveObj; - -typedef struct { - char key[TSDB_PARTITION_KEY_LEN]; - int64_t dbUid; - int64_t offset; -} SMqOffsetObj; - -int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset); -void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset); - typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7273e13317..05156e1427 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -18,6 +18,7 @@ #include "mndPrivilege.h" #include "mndVgroup.h" #include "mndShow.h" +#include "mndDb.h" #include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" @@ -124,7 +125,7 @@ void mndRebCntDec() { } } -static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser) { +static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { @@ -139,6 +140,22 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * return -1; } + if(enableReplay){ + if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){ + return TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; + }else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { + SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); + if (pDb == NULL) { + mndReleaseTopic(pMnode, pTopic); + return -1; + } + if (pDb->cfg.numOfVgroups != 1) { + return TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; + } + mndReleaseDb(pMnode, pDb); + } + } + mndTransSetDbName(pTrans, pOneTopic, NULL); if(mndTransCheckConflict(pMnode, pTrans) != 0){ mndReleaseTopic(pMnode, pTopic); @@ -177,7 +194,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { if (pTrans == NULL) { goto FAIL; } - if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user) != 0){ + if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0){ goto FAIL; } @@ -697,7 +714,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto _over; } - code = validateTopics(pTrans, pTopicList, pMnode, pMsg->info.conn.user); + code = validateTopics(pTrans, pTopicList, pMnode, pMsg->info.conn.user, subscribe.enableReplay); if (code != TSDB_CODE_SUCCESS) { goto _over; } diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 62b5cb00e6..481495cbe5 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -330,24 +330,6 @@ void dumpSubscribe(SSdb *pSdb, SJson *json) { } } -void dumpOffset(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonAddArrayToObject(json, "offsets"); - - while (1) { - SMqOffsetObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToArray(items, item); - tjsonAddStringToObject(item, "key", pObj->key); - tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); - tjsonAddStringToObject(item, "offset", i642str(pObj->offset)); - sdbRelease(pSdb, pObj); - } -} - void dumpStream(SSdb *pSdb, SJson *json) { void *pIter = NULL; SJson *items = tjsonAddArrayToObject(json, "streams"); @@ -608,7 +590,7 @@ void mndDumpSdb() { dumpTopic(pSdb, json); dumpConsumer(pSdb, json); dumpSubscribe(pSdb, json); - dumpOffset(pSdb, json); +// dumpOffset(pSdb, json); dumpStream(pSdb, json); dumpAcct(pSdb, json); dumpAuth(pSdb, json); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 94fd6027c0..4b5cfc830f 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -298,11 +298,6 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->version, pNewTopic->version); - /*taosWLockLatch(&pOldTopic->lock);*/ - - // TODO handle update - - /*taosWUnLockLatch(&pOldTopic->lock);*/ return 0; } @@ -320,23 +315,6 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { sdbRelease(pSdb, pTopic); } -static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { - int32_t contLen = sizeof(SDDropTopicReq); - - SDDropTopicReq *pDrop = taosMemoryCalloc(1, contLen); - if (pDrop == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pDrop->head.contLen = htonl(contLen); - pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pTopic->name, TSDB_TOPIC_FNAME_LEN); - pDrop->tuid = htobe64(pTopic->uid); - - return pDrop; -} - static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { terrno = TSDB_CODE_MND_INVALID_TOPIC; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e15f5f911d..ee0f0e2eeb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -213,7 +213,7 @@ typedef struct STqReader { SPackedData msg; SSubmitReq2 submit; int32_t nextBlk; - int64_t lastBlkUid; + int64_t lastTs; SWalReader *pWalReader; SMeta *pVnodeMeta; SHashObj *tbIdHash; @@ -241,6 +241,7 @@ bool tqNextBlockInWal(STqReader *pReader, const char *idstr); bool tqNextBlockImpl(STqReader *pReader, const char *idstr); SWalReader *tqGetWalReader(STqReader *pReader); SSDataBlock *tqGetResultBlock(STqReader *pReader); +int64_t tqGetResultBlockTime(STqReader *pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ee96e602d8..5d4ea8699b 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -47,6 +47,7 @@ typedef struct STqOffsetStore STqOffsetStore; // tqPush #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) #define STREAM_EXEC_TASK_STATUS_CHECK_ID (-2) +#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) // tqExec typedef struct { @@ -91,6 +92,10 @@ typedef struct { STqExecHandle execHandle; // exec SRpcMsg* msg; tq_handle_status status; + + // for replay + SSDataBlock* block; + int64_t blockTime; } STqHandle; struct STQ { @@ -108,17 +113,13 @@ struct STQ { SStreamMeta* pStreamMeta; }; -typedef struct { - int32_t size; -} STqOffsetHead; - int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); void tqDestroyTqHandle(void* data); // tqRead int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); -int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); +int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index adf3abe4d9..2fda70bcc1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -82,6 +82,9 @@ void tqDestroyTqHandle(void* data) { taosMemoryFree(pData->msg); pData->msg = NULL; } + if (pData->block != NULL){ + blockDataDestroy(pData->block); + } } static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 11bb737225..5e67f3c3ac 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -37,11 +37,9 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { int32_t vgId = TD_VID(pStore->pTq->pVnode); int64_t code = 0; - - STqOffsetHead head = {0}; - + int32_t size = 0; while (1) { - if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if (code == 0) { break; } else { @@ -49,7 +47,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { } } - int32_t size = htonl(head.size); void* pMemBuf = taosMemoryCalloc(1, size); if (pMemBuf == NULL) { tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); @@ -175,11 +172,11 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { return -1; } - int32_t totLen = sizeof(STqOffsetHead) + bodyLen; + int32_t totLen = INT_BYTES + bodyLen; void* buf = taosMemoryCalloc(1, totLen); - void* abuf = POINTER_SHIFT(buf, sizeof(STqOffsetHead)); + void* abuf = POINTER_SHIFT(buf, INT_BYTES); - ((STqOffsetHead*)buf)->size = htonl(bodyLen); + *(int32_t*)buf = bodyLen; SEncoder encoder; tEncoderInit(&encoder, abuf, bodyLen); tEncodeSTqOffset(&encoder, pOffset); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 4d470ee5b6..65914cc70e 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -475,6 +475,10 @@ SSDataBlock* tqGetResultBlock (STqReader* pReader) { return pReader->pResBlock; } +int64_t tqGetResultBlockTime(STqReader *pReader){ + return pReader->lastTs; +} + bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { return false; @@ -641,7 +645,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; - pReader->lastBlkUid = uid; + pReader->lastTs = pSubmitTbData->ctimeMs; pBlock->info.id.uid = uid; pBlock->info.version = pReader->msg.ver; @@ -783,9 +787,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas } int32_t sversion = pSubmitTbData->sver; - int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; - pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index cbe3ffee9e..705fb86fab 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -64,7 +64,23 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in return 0; } -int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { +int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res){ + uint64_t ts = 0; + qStreamSetOpen(task); + + tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); + int32_t code = qExecTask(task, res, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; + return -1; + } + + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, *res); + return 0; +} + +int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) { const int32_t MAX_ROWS_TO_RETURN = 4096; int32_t vgId = TD_VID(pTq->pVnode); @@ -80,34 +96,63 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs while (1) { SSDataBlock* pDataBlock = NULL; - uint64_t ts = 0; - qStreamSetOpen(task); - - tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - code = qExecTask(task, &pDataBlock, &ts); - if (code != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); - terrno = code; - return -1; - } - - tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, - pDataBlock); - // current scan should be stopped asap, since the rebalance occurs. - if (pDataBlock == NULL) { - break; - } - - code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d, failed to add block to rsp msg", vgId); + code = getDataBlock(task, pHandle, vgId, &pDataBlock); + if (code != 0){ return code; } - pRsp->blockNum++; - totalRows += pDataBlock->info.rows; - if (totalRows >= MAX_ROWS_TO_RETURN) { + if(pRequest->enableReplay){ + if(IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL){ + blockDataDestroy(pHandle->block); + pHandle->block = NULL; + } + if(pHandle->block == NULL){ + if (pDataBlock == NULL) { + break; + } + STqOffsetVal offset = {0}; + qStreamExtractOffset(task, &offset); + pHandle->block = createDataBlock(); + copyDataBlock(pHandle->block, pDataBlock); + pHandle->blockTime = offset.ts; + code = getDataBlock(task, pHandle, vgId, &pDataBlock); + if (code != 0){ + return code; + } + } + + code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, failed to add block to rsp msg", vgId); + return code; + } + + pRsp->blockNum++; + if (pDataBlock == NULL) { + break; + } + copyDataBlock(pHandle->block, pDataBlock); + + STqOffsetVal offset = {0}; + qStreamExtractOffset(task, &offset); + pRsp->sleepTime = offset.ts - pHandle->blockTime; + pHandle->blockTime = offset.ts; break; + }else{ + if (pDataBlock == NULL) { + break; + } + code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, failed to add block to rsp msg", vgId); + return code; + } + + pRsp->blockNum++; + totalRows += pDataBlock->info.rows; + if (totalRows >= MAX_ROWS_TO_RETURN) { + break; + } } } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 04695c1f63..a4c3d395e3 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -15,8 +15,6 @@ #include "tq.h" -#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) - static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); @@ -142,7 +140,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, tqInitDataRsp(&dataRsp, *pOffset); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index c72ecd4824..a6a2e128d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -131,6 +131,7 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqGetResultBlock = tqGetResultBlock; pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; + pTq->tqGetResultBlockTime = tqGetResultBlockTime; } void initStateStoreAPI(SStateStore* pStore) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 474128007a..913e246f63 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1862,6 +1862,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { // curVersion move to next tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion); + // use ts to pass time when replay, because ts not used if type is log + pTaskInfo->streamInfo.currentOffset.ts = pAPI->tqReaderFn.tqGetResultBlockTime(pInfo->tqReader); + if (hasResult) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9832720994..dd51b99cd1 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -654,6 +654,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE, "Topic num out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE, "Group num out of range 100") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SAME_COMMITTED_VALUE, "Same committed value") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP, "Replay need only one vgroup if subscribe super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled if subscribe db or stable") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") diff --git a/utils/test/c/tmqOffset.c b/utils/test/c/tmqOffset.c index 7225cb87bd..9699e71f24 100644 --- a/utils/test/c/tmqOffset.c +++ b/utils/test/c/tmqOffset.c @@ -7,18 +7,14 @@ #include "tlog.h" #include "tmsg.h" -typedef struct { - int32_t size; -} STqOffsetHead; - int32_t tqOffsetRestoreFromFile(const char* fname) { TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile != NULL) { - STqOffsetHead head = {0}; int32_t code; while (1) { - if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + int32_t size = 0; + if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if (code == 0) { break; } else { @@ -26,7 +22,6 @@ int32_t tqOffsetRestoreFromFile(const char* fname) { return -1; } } - int32_t size = htonl(head.size); void* memBuf = taosMemoryCalloc(1, size); if (memBuf == NULL) { printf("memBuf == NULL\n"); From a30beeee6de93bb4bdf69940465af678ae8e3cf7 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Sun, 8 Oct 2023 16:46:03 +0800 Subject: [PATCH 008/177] Update tmqVnodeTransform.py --- tests/system-test/7-tmq/tmqVnodeTransform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index 3ebf48b93a..aab94bc7a2 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -331,9 +331,9 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") def run(self): - # self.prepareTestEnv() - # self.tmqCase1() - # self.tmqCase2() + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() self.prepareTestEnv() self.tmqCase3() From 5f7b6f19baeac0a5d4d30895f80e11ece6246328 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 8 Oct 2023 19:05:59 +0800 Subject: [PATCH 009/177] feat:[TD-26056] add replay logic --- source/client/src/clientTmq.c | 1 + source/dnode/vnode/inc/vnode.h | 3 +- source/libs/executor/src/projectoperator.c | 3 +- tests/system-test/7-tmq/replay.py | 320 +++++++++++++++++++++ 4 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/7-tmq/replay.py diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 50b8eb1eca..252959ecd8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1532,6 +1532,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->head.vgId = pVg->vgId; pReq->useSnapshot = tmq->useSnapshot; pReq->reqId = generateRequestId(); + pReq->enableReplay = tmq->replayEnable; } SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 122f6733fb..787b717015 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -216,7 +216,7 @@ typedef struct STqReader { SPackedData msg; SSubmitReq2 submit; int32_t nextBlk; - int64_t lastTs; + int64_t lastBlkUid; SWalReader *pWalReader; SMeta *pVnodeMeta; SHashObj *tbIdHash; @@ -226,6 +226,7 @@ typedef struct STqReader { int64_t cachedSchemaUid; SSchemaWrapper *pSchemaWrapper; SSDataBlock *pResBlock; + int64_t lastTs; } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 00b246afad..227960ab22 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -110,7 +110,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder; - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM || pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { pInfo->mergeDataBlocks = false; } else { if (!pProjPhyNode->ignoreGroupId) { @@ -330,6 +330,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } + qDebug("project return %d", pProjectInfo->mergeDataBlocks); if (pProjectInfo->mergeDataBlocks) { if (pRes->info.rows > 0) { pFinalRes->info.id.groupId = 0; // clear groupId diff --git a/tests/system-test/7-tmq/replay.py b/tests/system-test/7-tmq/replay.py new file mode 100644 index 0000000000..7eee6743a7 --- /dev/null +++ b/tests/system-test/7-tmq/replay.py @@ -0,0 +1,320 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + time.sleep(1) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"], \ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase8(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 8: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db8', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 1, \ + 'rowsPerTbl': 10, \ + 'batchNum': 1, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest,\ + enable.replay:true' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume 0 processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume 0 result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + # tdLog.info("start consume 1 processor") + # self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + # tdLog.sleep(2) + # + # tdLog.info("start one new thread to insert data") + # parameterDict['actionType'] = actionType.INSERT_DATA + # prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + # prepareEnvThread.start() + # prepareEnvThread.join() + # + # tdLog.info("start to check consume 0 and 1 result") + # expectRows = 2 + # resultList = self.selectConsumeResult(expectRows) + # totalConsumeRows = 0 + # for i in range(expectRows): + # totalConsumeRows += resultList[i] + # + # if totalConsumeRows != expectrowcnt: + # tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + # tdLog.exit("tmq consume rows error!") + # + # tdLog.info("start consume 2 processor") + # self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + # tdLog.sleep(2) + # + # tdLog.info("start one new thread to insert data") + # parameterDict['actionType'] = actionType.INSERT_DATA + # prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + # prepareEnvThread.start() + # prepareEnvThread.join() + # + # tdLog.info("start to check consume 0 and 1 and 2 result") + # expectRows = 3 + # resultList = self.selectConsumeResult(expectRows) + # totalConsumeRows = 0 + # for i in range(expectRows): + # totalConsumeRows += resultList[i] + # + # if totalConsumeRows != expectrowcnt*2: + # tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) + # tdLog.exit("tmq consume rows error!") + # + # tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 8 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase8(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 6ad518d61de8f142e4160d4aff15b15d43bd0ce2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 9 Oct 2023 13:21:32 +0800 Subject: [PATCH 010/177] enhance: pseduo column coverted to column when necessary --- source/libs/parser/src/parTranslater.c | 96 ++++++++++++++++---------- 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0fc57d3729..2a55a5fcd3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1571,25 +1571,6 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { return TSDB_CODE_SUCCESS; } -static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } - } else { - SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); - STableNode* pTable = NULL; - pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsIndefiniteRowsFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1726,20 +1707,6 @@ static int32_t translateForbidFillFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } -static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); - } - if (beforeWindow(pCxt->currClause)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", - pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsForbidStreamFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1978,10 +1945,65 @@ static int32_t rewriteSystemInfoFunc(STranslateContext* pCxt, SNode** pNode) { return TSDB_CODE_PAR_INTERNAL_ERROR; } -static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { +static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode** ppNode) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (pCol == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SExprNode* pOldExpr = (SExprNode*)(*ppNode); + pCol->node.resType = pOldExpr->resType; + strcpy(pCol->node.aliasName, pOldExpr->aliasName); + strcpy(pCol->node.userAlias, pOldExpr->userAlias); + strcpy(pCol->colName, pOldExpr->aliasName); + + nodesDestroyNode(*ppNode); + *ppNode = (SNode*)pCol; + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (!isSelectStmt(pCxt->pCurrStmt)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); + } + if (((SSelectStmt*)pCxt->pCurrStmt)->pWindow && beforeWindow(pCxt->currClause)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", + pFunc->functionName); + } + if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { + replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + return translateColumn(pCxt, (SColumnNode**)ppNode); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateScanPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (0 == LIST_LENGTH(pFunc->pParameterList)) { + if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } + if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + return translateColumn(pCxt, (SColumnNode**)ppNode); + } + } else { + SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + STableNode* pTable = NULL; + pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); + if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { - code = translateScanPseudoColumnFunc(pCxt, pFunc); + code = translateScanPseudoColumnFunc2(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateIndefiniteRowsFunc(pCxt, pFunc); @@ -1990,7 +2012,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p code = translateForbidFillFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindowPseudoColumnFunc(pCxt, pFunc); + code = translateWindowPseudoColumnFunc2(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); @@ -2082,7 +2104,7 @@ static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pF if (fmIsClientPseudoColumnFunc((*pFunc)->funcId)) { return rewriteClientPseudoColumnFunc(pCxt, (SNode**)pFunc); } - return translateNormalFunction(pCxt, *pFunc); + return translateNormalFunction(pCxt, (SNode**)pFunc); } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) { From 56534a7b0d47b2601a960dee5a985f97b5222dc3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 9 Oct 2023 16:22:44 +0800 Subject: [PATCH 011/177] enhance: scan and window pseudo column --- source/libs/parser/src/parTranslater.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b341e4e5eb..0228a7fded 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1934,6 +1934,9 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (!isSelectStmt(pCxt->pCurrStmt)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); } @@ -1942,27 +1945,38 @@ static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** pFunc->functionName); } if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - return translateColumn(pCxt, (SColumnNode**)ppNode); + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + return pCxt->errCode; } return TSDB_CODE_SUCCESS; } static int32_t translateScanPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (0 == LIST_LENGTH(pFunc->pParameterList)) { if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { - replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - return translateColumn(pCxt, (SColumnNode**)ppNode); + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + return pCxt->errCode; } } else { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); STableNode* pTable = NULL; pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } From 44c3bc99e09c939eac667ed295b90be3724eefe5 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 9 Oct 2023 16:56:21 +0800 Subject: [PATCH 012/177] feat: add interp pseudo column conversion to column --- source/libs/parser/src/parTranslater.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0228a7fded..4e0979ddaa 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -263,6 +263,8 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMet static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); +static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode** ppNode); + static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; } static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } @@ -1619,7 +1621,8 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } -static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } @@ -1631,6 +1634,24 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctio return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, "%s is not allowed in where clause", pFunc->functionName); } + + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + SNode* pNode = NULL; + bool bFound = false; + FOREACH(pNode, pSelect->pProjectionList) { + if (nodeType(pNode) == QUERY_NODE_FUNCTION && strcasecmp(((SFunctionNode*)pNode)->functionName, "interp") == 0) { + bFound = true; + break; + } + } + if (!bFound) { + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + return pCxt->errCode; + } return TSDB_CODE_SUCCESS; } @@ -2017,7 +2038,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateInterpFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateInterpPseudoColumnFunc(pCxt, pFunc); + code = translateInterpPseudoColumnFunc(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); From 33045e63ae88bd4335ec203f6e6e22a619684498 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 17:35:40 +0800 Subject: [PATCH 013/177] feat:[TD-26056] add replay logic --- source/client/src/clientTmq.c | 17 ++ source/dnode/vnode/src/tq/tqScan.c | 21 +- source/dnode/vnode/src/tq/tqUtil.c | 6 +- source/libs/executor/src/projectoperator.c | 1 - tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/replay.py | 33 +-- tests/system-test/7-tmq/tmq_replay.py | 39 +++ utils/test/c/CMakeLists.txt | 9 + utils/test/c/replay_test.c | 323 +++++++++++++++++++++ utils/test/c/tmqSim.c | 3 +- 10 files changed, 418 insertions(+), 35 deletions(-) create mode 100644 tests/system-test/7-tmq/tmq_replay.py create mode 100644 utils/test/c/replay_test.c diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 252959ecd8..e398441979 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -724,6 +724,17 @@ void tmqAssignAskEpTask(void* param, void* tmrId) { taosMemoryFree(param); } +void tmqReplayTask(void* param, void* tmrId) { + int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if(tmq == NULL) goto END; + + tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); +END: + taosMemoryFree(param); +} + void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT); @@ -1144,6 +1155,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.autoCommit = tmq->autoCommit; req.autoCommitInterval = tmq->autoCommitInterval; req.resetOffsetCfg = tmq->resetOffsetCfg; + req.enableReplay = tmq->replayEnable; for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1823,6 +1835,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { if(tmq->replayEnable){ pVg->blockReceiveTs = taosGetTimestampMs(); pVg->blockSleepForReplay = pRsp->rsp.sleepTime; + if(pVg->blockSleepForReplay > 0){ + int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t)); + *pRefId1 = tmq->refId; + taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, pRefId1, tmqMgmt.timer); + } } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 705fb86fab..01866ef893 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -112,8 +112,9 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* } STqOffsetVal offset = {0}; qStreamExtractOffset(task, &offset); - pHandle->block = createDataBlock(); - copyDataBlock(pHandle->block, pDataBlock); + pHandle->block = createOneDataBlock(pDataBlock, true); +// pHandle->block = createDataBlock(); +// copyDataBlock(pHandle->block, pDataBlock); pHandle->blockTime = offset.ts; code = getDataBlock(task, pHandle, vgId, &pDataBlock); if (code != 0){ @@ -129,14 +130,16 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pRsp->blockNum++; if (pDataBlock == NULL) { - break; - } - copyDataBlock(pHandle->block, pDataBlock); + blockDataDestroy(pHandle->block); + pHandle->block = NULL; + }else{ + copyDataBlock(pHandle->block, pDataBlock); - STqOffsetVal offset = {0}; - qStreamExtractOffset(task, &offset); - pRsp->sleepTime = offset.ts - pHandle->blockTime; - pHandle->blockTime = offset.ts; + STqOffsetVal offset = {0}; + qStreamExtractOffset(task, &offset); + pRsp->sleepTime = offset.ts - pHandle->blockTime; + pHandle->blockTime = offset.ts; + } break; }else{ if (pDataBlock == NULL) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index a4c3d395e3..215f8d3cb2 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -40,11 +40,11 @@ void tqUpdateNodeStage(STQ* pTq) { tqDebug("vgId:%d update the meta stage to be:%"PRId64, pTq->pStreamMeta->vgId, pTq->pStreamMeta->stage); } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset, bool withTbName) { pRsp->reqOffset = pOffset; pRsp->rspOffset = pOffset; - pRsp->withTbName = 1; + pRsp->withTbName = withTbName; pRsp->withSchema = 1; pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -177,7 +177,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t vgId = TD_VID(pTq->pVnode); SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, *offset); + tqInitTaosxRsp(&taosxRsp, *offset, pRequest->withTbName); if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 227960ab22..f60890ecca 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -330,7 +330,6 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } - qDebug("project return %d", pProjectInfo->mergeDataBlocks); if (pProjectInfo->mergeDataBlocks) { if (pRes->info.rows > 0) { pFinalRes->info.id.groupId = 0; // clear groupId diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 36b8fded81..b5856fea63 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -159,6 +159,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDataPrecisionUnit.py diff --git a/tests/system-test/7-tmq/replay.py b/tests/system-test/7-tmq/replay.py index 7eee6743a7..bbda8600fb 100644 --- a/tests/system-test/7-tmq/replay.py +++ b/tests/system-test/7-tmq/replay.py @@ -110,7 +110,7 @@ class TDTestCase: tdLog.info(shellCmd) os.system(shellCmd) - def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + def create_database(self,tsql, dbName,dropFlag=1,vgroups=1,replica=1): if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) @@ -149,21 +149,12 @@ class TDTestCase: t = time.time() startTs = int(round(t * 1000)) - #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) - rowsOfSql = 0 - for i in range(ctbNum): - sql += " %s_%d values "%(stbName,i) - for j in range(rowsPerTbl): - sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) - rowsOfSql += 1 - if ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): - tsql.execute(sql) - time.sleep(1) - rowsOfSql = 0 - if j < rowsPerTbl - 1: - sql = "insert into %s_%d values " %(stbName,i) - else: - sql = "insert into " + for j in range(rowsPerTbl): + for i in range(ctbNum): + sql += " %s_%d values (%d, %d, 'tmqrow_%d') "%(stbName, i, startTs + j + i, j+i, j+i) + tsql.execute(sql) + time.sleep(1) + sql = "insert into " #end sql if sql != pre_insert: #print("insert sql:%s"%sql) @@ -199,10 +190,10 @@ class TDTestCase: 'actionType': 0, \ 'dbName': 'db8', \ 'dropFlag': 1, \ - 'vgroups': 4, \ + 'vgroups': 1, \ 'replica': 1, \ 'stbName': 'stb1', \ - 'ctbNum': 1, \ + 'ctbNum': 2, \ 'rowsPerTbl': 10, \ 'batchNum': 1, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 @@ -223,7 +214,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] * 2 topicList = topicFromStb1 ifcheckdata = 0 ifManualCommit = 1 @@ -247,8 +238,8 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != 0: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") # tdLog.info("start consume 1 processor") diff --git a/tests/system-test/7-tmq/tmq_replay.py b/tests/system-test/7-tmq/tmq_replay.py new file mode 100644 index 0000000000..1e19d58516 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_replay.py @@ -0,0 +1,39 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + buildPath = tdCom.getBuildPath() + + cmdStr1 = '%s/build/bin/replay_test'%(buildPath) + tdLog.info(cmdStr1) + result = os.system(cmdStr1) + + if result != 0: + tdLog.exit("tmq_replay error!") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 343e3d8454..db5eb21ad8 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(get_db_name_test get_db_name_test.c) add_executable(tmq_offset tmqOffset.c) add_executable(tmq_offset_test tmq_offset_test.c) add_executable(varbinary_test varbinary_test.c) +add_executable(replay_test replay_test.c) if(${TD_LINUX}) add_executable(tsz_test tsz_test.c) @@ -57,6 +58,14 @@ target_link_libraries( PUBLIC os ) +target_link_libraries( + replay_test + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) + target_link_libraries( write_raw_block_test PUBLIC taos diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c new file mode 100644 index 0000000000..1fbaac0796 --- /dev/null +++ b/utils/test/c/replay_test.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "types.h" + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "g1"); + tmq_conf_set(conf, "client.id", "c1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.replay", "true"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +void test_vgroup_error(TAOS* pConn){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 2 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + ASSERT(tmq_subscribe(tmq, topic_list) != 0); + tmq_list_destroy(topic_list); + tmq_consumer_close(tmq); +} + +void test_stable_db_error(TAOS* pConn){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 1 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic t1 as stable d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + ASSERT(tmq_subscribe(tmq, topic_list) != 0); + tmq_list_destroy(topic_list); + tmq_consumer_close(tmq); + + pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic t1 as database d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + topic_list = tmq_list_new(); + tmq_list_append(topic_list, "t1"); + tmq = build_consumer(); + ASSERT(tmq_subscribe(tmq, topic_list) != 0); + tmq_list_destroy(topic_list); + tmq_consumer_close(tmq); +} + +void insert_with_sleep(TAOS* pConn, int32_t* interval, int32_t len){ + for(int i = 0; i < len; i++){ + TAOS_RES* pRes = taos_query(pConn, "insert into d1.table1 (ts, c1) values (now, 1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + taosMsleep(interval[i]); + } +} + +void insert_with_sleep_multi(TAOS* pConn, int32_t* interval, int32_t len){ + for(int i = 0; i < len; i++){ + TAOS_RES* pRes = taos_query(pConn, "insert into d1.table1 (ts, c1) values (now, 1) (now+1s, 2) d1.table2 (ts, c1) values (now, 1) (now+1s, 2)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + taosMsleep(interval[i]); + } +} + +void test_case1(TAOS* pConn, int32_t* interval, int32_t len){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 2 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table1 using d1.s1 tags(1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + insert_with_sleep(pConn, interval, len); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.table1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + tmq_list_destroy(topic_list); + + int32_t timeout = 5000; + + int64_t t = 0; + int32_t totalRows = 0; + char buf[1024] = {0}; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, timeout); + if (tmqmessage) { + if(t != 0){ + ASSERT(taosGetTimestampMs() - t >= interval[totalRows - 1]); + } + t = taosGetTimestampMs(); + + TAOS_ROW row = taos_fetch_row(tmqmessage); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); + int32_t numOfFields = taos_field_count(tmqmessage); + const char* tbName = tmq_get_table_name(tmqmessage); + taos_print_row(buf, row, fields, numOfFields); + + printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + totalRows++; + taos_free_result(tmqmessage); + } else { + break; + } + } + + ASSERT(totalRows == len); + tmq_consumer_close(tmq); +} + +void test_case2(TAOS* pConn, int32_t* interval, int32_t len, tsem_t* sem){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 1 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table1 using d1.s1 tags(1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table2 using d1.s1 tags(2)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + insert_with_sleep_multi(pConn, interval, len); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + tmq_list_destroy(topic_list); + + int32_t timeout = 5000; + + int64_t t = 0; + int32_t totalRows = 0; + char buf[1024] = {0}; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, timeout); + if (tmqmessage) { + if(t != 0 && totalRows % 4 == 0){ + ASSERT(taosGetTimestampMs() - t >= interval[totalRows/4 - 1]); + } + t = taosGetTimestampMs(); + + while(1){ + TAOS_ROW row = taos_fetch_row(tmqmessage); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); + int32_t numOfFields = taos_field_count(tmqmessage); + const char* tbName = tmq_get_table_name(tmqmessage); + taos_print_row(buf, row, fields, numOfFields); + + printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + totalRows++; + } + + taos_free_result(tmqmessage); + + if(totalRows == len * 4){ + taosSsleep(1); + tsem_post(sem); + } + } else { + break; + } + } + + ASSERT(totalRows == len * 4 + 1); + tmq_consumer_close(tmq); +} + +void* insertThreadFunc(void* param) { + tsem_t* sem = (tsem_t*)param; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + + tsem_wait(sem); + + TAOS_RES* pRes = taos_query(pConn, "insert into d1.table1 (ts, c1) values (now, 11)"); + ASSERT(taos_errno(pRes) == 0); + printf("insert data again\n"); + taos_free_result(pRes); + taos_close(pConn); + return NULL; +} + +int main(int argc, char* argv[]) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + test_vgroup_error(pConn); + test_stable_db_error(pConn); + + tsem_t sem; + tsem_init(&sem, 0, 0); + TdThread thread; + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + // pthread_create one thread to consume + taosThreadCreate(&thread, &thattr, insertThreadFunc, (void*)(&sem)); + + int32_t interval[5] = {1000, 200, 3000, 40, 500}; + test_case1(pConn, interval, sizeof(interval)/sizeof(int32_t)); + printf("test_case1 success\n"); + test_case2(pConn, interval, sizeof(interval)/sizeof(int32_t), &sem); + taos_close(pConn); + + taosThreadJoin(thread, NULL); + taosThreadClear(&thread); + tsem_destroy(&sem); + return 0; +} diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 6b774b3eff..34f4a9d094 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -621,10 +621,11 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn taos_print_row(buf, row, fields, numOfFields); if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); + taosFprintfFile(g_fp, "%lld tbname:%s, rows[%d]: %s\n", taosGetTimestampMs(), (tbName != NULL ? tbName : "null table"), totalRows, buf); // if (0 != g_stConfInfo.saveRowFlag) { // saveConsumeContentToTbl(pInfo, buf); // } +// taosFsyncFile(g_fp); } totalRows++; From 3e2e924e9867b41691a6b547485f626d9b590278 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 18:36:39 +0800 Subject: [PATCH 014/177] feat:[TD-26056] add replay logic --- source/dnode/mnode/impl/src/mndConsumer.c | 31 +++++++++++++++-------- source/dnode/vnode/src/tq/tqUtil.c | 6 ++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 05156e1427..c9ee66d3a0 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -126,31 +126,37 @@ void mndRebCntDec() { } static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { - int32_t numOfTopics = taosArrayGetSize(pTopicList); + SMqTopicObj *pTopic = NULL; + int32_t code = 0; + int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { char *pOneTopic = taosArrayGetP(pTopicList, i); - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOneTopic); + pTopic = mndAcquireTopic(pMnode, pOneTopic); if (pTopic == NULL) { // terrno has been set by callee function - return -1; + code = -1; + goto FAILED; } if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) { - mndReleaseTopic(pMnode, pTopic); - return -1; + code = -1; + goto FAILED; } if(enableReplay){ if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){ - return TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; + code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; + goto FAILED; }else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); if (pDb == NULL) { - mndReleaseTopic(pMnode, pTopic); - return -1; + code = -1; + goto FAILED; } if (pDb->cfg.numOfVgroups != 1) { - return TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; + mndReleaseDb(pMnode, pDb); + code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; + goto FAILED; } mndReleaseDb(pMnode, pDb); } @@ -158,13 +164,16 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * mndTransSetDbName(pTrans, pOneTopic, NULL); if(mndTransCheckConflict(pMnode, pTrans) != 0){ - mndReleaseTopic(pMnode, pTopic); - return -1; + code = -1; + goto FAILED; } mndReleaseTopic(pMnode, pTopic); } return 0; +FAILED: + mndReleaseTopic(pMnode, pTopic); + return code; } static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 215f8d3cb2..a4c3d395e3 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -40,11 +40,11 @@ void tqUpdateNodeStage(STQ* pTq) { tqDebug("vgId:%d update the meta stage to be:%"PRId64, pTq->pStreamMeta->vgId, pTq->pStreamMeta->stage); } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset, bool withTbName) { +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { pRsp->reqOffset = pOffset; pRsp->rspOffset = pOffset; - pRsp->withTbName = withTbName; + pRsp->withTbName = 1; pRsp->withSchema = 1; pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -177,7 +177,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t vgId = TD_VID(pTq->pVnode); SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, *offset, pRequest->withTbName); + tqInitTaosxRsp(&taosxRsp, *offset); if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { From d10915dce6a3cced5024c307834650f7ee19191e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 19:03:13 +0800 Subject: [PATCH 015/177] feat:[TD-26056] add replay logic --- utils/test/c/replay_test.c | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c index 1fbaac0796..df493ce2ac 100644 --- a/utils/test/c/replay_test.c +++ b/utils/test/c/replay_test.c @@ -281,6 +281,93 @@ void test_case2(TAOS* pConn, int32_t* interval, int32_t len, tsem_t* sem){ tmq_consumer_close(tmq); } +void test_case3(TAOS* pConn, int32_t* interval, int32_t len){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 1 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table1 using d1.s1 tags(1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table2 using d1.s1 tags(2)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + insert_with_sleep_multi(pConn, interval, len); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + + int32_t timeout = 5000; + + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, timeout); + taos_free_result(tmqmessage); + + tmq_consumer_close(tmq); + + tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + + int64_t t = 0; + int32_t totalRows = 0; + char buf[1024] = {0}; + while (1) { + tmqmessage = tmq_consumer_poll(tmq, timeout); + if (tmqmessage) { + if(t != 0 && totalRows % 4 == 0){ + ASSERT(taosGetTimestampMs() - t >= interval[totalRows/4 - 1]); + } + t = taosGetTimestampMs(); + + while(1){ + TAOS_ROW row = taos_fetch_row(tmqmessage); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); + int32_t numOfFields = taos_field_count(tmqmessage); + const char* tbName = tmq_get_table_name(tmqmessage); + taos_print_row(buf, row, fields, numOfFields); + + printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + totalRows++; + } + + taos_free_result(tmqmessage); + } else { + break; + } + } + + ASSERT(totalRows == len * 4); + + tmq_consumer_close(tmq); + tmq_list_destroy(topic_list); +} + void* insertThreadFunc(void* param) { tsem_t* sem = (tsem_t*)param; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -314,6 +401,8 @@ int main(int argc, char* argv[]) { test_case1(pConn, interval, sizeof(interval)/sizeof(int32_t)); printf("test_case1 success\n"); test_case2(pConn, interval, sizeof(interval)/sizeof(int32_t), &sem); + printf("test_case2 success\n"); + test_case3(pConn, interval, sizeof(interval)/sizeof(int32_t)); taos_close(pConn); taosThreadJoin(thread, NULL); From b12c734f096b348143a475969c138ee841b9c2a9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 21:14:32 +0800 Subject: [PATCH 016/177] feat:[TD-26056] add replay logic --- utils/test/c/tmqSim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 34f4a9d094..14e30008fe 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -621,7 +621,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn taos_print_row(buf, row, fields, numOfFields); if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "%lld tbname:%s, rows[%d]: %s\n", taosGetTimestampMs(), (tbName != NULL ? tbName : "null table"), totalRows, buf); + taosFprintfFile(g_fp, "time:%" PRId64 " tbname:%s, rows[%d]: %s\n", taosGetTimestampMs(), (tbName != NULL ? tbName : "null table"), totalRows, buf); // if (0 != g_stConfInfo.saveRowFlag) { // saveConsumeContentToTbl(pInfo, buf); // } From b8fe279a855edb5edabfe5608a970d19d42caf65 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 07:44:29 +0800 Subject: [PATCH 017/177] fix: renanme function --- source/libs/parser/src/parTranslater.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4e0979ddaa..201111e425 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1953,7 +1953,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_SUCCESS; } -static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1976,7 +1976,7 @@ static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** return TSDB_CODE_SUCCESS; } -static int32_t translateScanPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -2008,7 +2008,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { - code = translateScanPseudoColumnFunc2(pCxt, ppNode); + code = translateScanPseudoColumnFunc(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateIndefiniteRowsFunc(pCxt, pFunc); @@ -2017,7 +2017,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateForbidFillFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindowPseudoColumnFunc2(pCxt, ppNode); + code = translateWindowPseudoColumnFunc(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); From da9d39bfe6ec09afa701325f975670421b5e2d12 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 08:31:23 +0800 Subject: [PATCH 018/177] fix: cancel future pseudo column translation function processing when rewritten to column --- source/libs/parser/src/parTranslater.c | 27 ++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 201111e425..e285c7a806 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1621,7 +1621,7 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } -static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1645,6 +1645,7 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** } } if (!bFound) { + *pRewriteToColumn = true; int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1953,7 +1954,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_SUCCESS; } -static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1966,6 +1967,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** pFunc->functionName); } if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { + *pRewriteToColumn = true; int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1976,7 +1978,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** return TSDB_CODE_SUCCESS; } -static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1986,6 +1988,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + *pRewriteToColumn = true; int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2008,7 +2011,11 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { - code = translateScanPseudoColumnFunc(pCxt, ppNode); + bool bRewriteToColumn = false; + code = translateScanPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); + if (bRewriteToColumn) { + return code; + } } if (TSDB_CODE_SUCCESS == code) { code = translateIndefiniteRowsFunc(pCxt, pFunc); @@ -2017,7 +2024,11 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateForbidFillFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindowPseudoColumnFunc(pCxt, ppNode); + bool bRewriteToColumn = false; + code = translateWindowPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); + if (bRewriteToColumn) { + return code; + } } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); @@ -2038,7 +2049,11 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateInterpFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateInterpPseudoColumnFunc(pCxt, ppNode); + bool bRewriteToColumn = false; + code = translateInterpPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); + if (bRewriteToColumn) { + return code; + } } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); From d42e819d2d89d00a43e10c32ba0a92f64821bffd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 09:02:37 +0800 Subject: [PATCH 019/177] feat:[TD-26056] add replay logic --- utils/test/c/replay_test.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c index df493ce2ac..c105670733 100644 --- a/utils/test/c/replay_test.c +++ b/utils/test/c/replay_test.c @@ -184,10 +184,9 @@ void test_case1(TAOS* pConn, int32_t* interval, int32_t len){ TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); int32_t numOfFields = taos_field_count(tmqmessage); - const char* tbName = tmq_get_table_name(tmqmessage); taos_print_row(buf, row, fields, numOfFields); - printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + printf("time:%" PRId64 " rows[%d]: %s\n", t, totalRows, buf); totalRows++; taos_free_result(tmqmessage); } else { @@ -259,10 +258,9 @@ void test_case2(TAOS* pConn, int32_t* interval, int32_t len, tsem_t* sem){ TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); int32_t numOfFields = taos_field_count(tmqmessage); - const char* tbName = tmq_get_table_name(tmqmessage); taos_print_row(buf, row, fields, numOfFields); - printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + printf("time:%" PRId64 " rows[%d]: %s\n", t, totalRows, buf); totalRows++; } @@ -349,10 +347,9 @@ void test_case3(TAOS* pConn, int32_t* interval, int32_t len){ TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); int32_t numOfFields = taos_field_count(tmqmessage); - const char* tbName = tmq_get_table_name(tmqmessage); taos_print_row(buf, row, fields, numOfFields); - printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + printf("time:%" PRId64 " rows[%d]: %s\n", t, totalRows, buf); totalRows++; } From bc2cf345fade4301cba25785e98817d5cd0751fb Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 10:44:13 +0800 Subject: [PATCH 020/177] fix: change error code after replace column and translate --- source/libs/parser/src/parTranslater.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e285c7a806..c1881e3d27 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1973,7 +1973,11 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** return code; } translateColumn(pCxt, (SColumnNode**)ppNode); - return pCxt->errCode; + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); + } else { + return TSDB_CODE_SUCCESS; + } } return TSDB_CODE_SUCCESS; } @@ -1994,7 +1998,11 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp return code; } translateColumn(pCxt, (SColumnNode**)ppNode); - return pCxt->errCode; + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } else { + return TSDB_CODE_SUCCESS; + } } } else { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); From 53c84a7245c952514c485e227f2827c96230df65 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 11:02:01 +0800 Subject: [PATCH 021/177] doc:add doc for replay --- docs/en/07-develop/07-tmq.mdx | 18 ++++++++++++++++++ docs/zh/07-develop/07-tmq.mdx | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index f833dbf439..a511eaa4c3 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -514,6 +514,24 @@ var consumer = new ConsumerBuilder(cfg).Build(); A consumer group is automatically created when multiple consumers are configured with the same consumer group ID. +Data replay function description: +- Subscription adds replay function, which replays according to the time of data writing. + For example, writing three pieces of data at the following time. + ```sql + 2023/09/22 00:00:00.000 + 2023/09/22 00:00:05.000 + 2023/09/22 00:00:08.000 + ``` + After subscribing to the first data for 5 seconds, the second data is returned, and after obtaining the second data for 3 seconds, the third data is returned. +- Only column subscriptions support data replay. + - Replay needs to ensure an independent timeline + - If it is a sub table subscription or a normal table subscription, only one vnode has data, ensuring a timeline. + - If subscribing to a super table, it is necessary to ensure that the DB has only one vnode, otherwise an error will be reported (because the data subscribed to on multiple vnodes is not on the same timeline). +- Super table and database subscriptions do not support replay +- Add the enable.replay parameter. True indicates that the subscription replay function is enabled, while false indicates that the subscription replay function is not enabled by default. +- Replay does not support progress saving, so when the replay parameter enable, auto commit will automatically close. +- Due to the processing time required for data replay, there is an error of tens of milliseconds in the accuracy of replay. + ## Subscribe to a Topic A single consumer can subscribe to multiple topics. diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 927d762829..fee0ec86c2 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -355,6 +355,7 @@ CREATE TOPIC topic_name [with meta] AS DATABASE db_name; | `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | | `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | | `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句） |默认关闭 | +| `enable.replay` | boolean | 是否开启数据回放功能 |默认关闭 | 对于不同编程语言，其设置方式如下： @@ -515,6 +516,24 @@ var consumer = new ConsumerBuilder(cfg).Build(); 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。 +数据回放功能说明： +- 订阅增加 replay 功能，按照数据写入的时间回放。 + 比如，如下时间写入三条数据 + ```sql + 2023/09/22 00:00:00.000 + 2023/09/22 00:00:05.000 + 2023/09/22 00:00:08.000 + ``` + 则订阅出第一条数据 5s 后返回第二条数据，获取第二条数据 3s 后返回第三条数据。 +- 仅列订阅支持数据回放 + - 回放需要保证独立时间线 + - 如果是子表订阅或者普通表订阅，只有一个vnode上有数据，保证是一个时间线 + - 如果超级表订阅，则需保证该 DB 只有一个vnode，否则报错（因为多个vnode上订阅出的数据不在一个时间线上） +- 超级表和库订阅不支持回放 +- 增加 enable.replay 参数，true表示开启订阅回放功能，false表示不开启订阅回放功能，默认不开启。 +- 回放不支持进度保存，所以回放参数 enable.replay = true 时，auto commit 自动关闭 +- 因为数据回放本身需要处理时间，所以回放的精度存在几十ms的误差 + ## 订阅 *topics* 一个 consumer 支持同时订阅多个 topic。 From 9113c3c3b6032e9d2d0c2d4c7e4592f80da2b672 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 11:53:54 +0800 Subject: [PATCH 022/177] fix:rollback removed code --- docs/en/07-develop/07-tmq.mdx | 1 + source/dnode/vnode/src/tq/tqRead.c | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index a511eaa4c3..86b2498345 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -356,6 +356,7 @@ You configure the following parameters when creating a consumer: | `enable.auto.commit` | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself | Default value is true | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | | `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | default value: false +| `enable.replay` | boolean | Specify whether data replay function enabled or not |default value: false | The method of specifying these parameters depends on the language used: diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 65914cc70e..3b052a3edd 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -788,6 +788,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas int32_t sversion = pSubmitTbData->sver; int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); From 1762a85127d8a16d6b34b51a4e64ca31981d421d Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 14:33:53 +0800 Subject: [PATCH 023/177] enhance: add test case --- tests/develop-test/2-query/pseudo_column.py | 81 +++++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 82 insertions(+) create mode 100644 tests/develop-test/2-query/pseudo_column.py diff --git a/tests/develop-test/2-query/pseudo_column.py b/tests/develop-test/2-query/pseudo_column.py new file mode 100644 index 0000000000..9f6366a3c6 --- /dev/null +++ b/tests/develop-test/2-query/pseudo_column.py @@ -0,0 +1,81 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TS-3904/TS-3005] pseudo column test case + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self._conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use pseudo_col") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists pseudo_col") + tdSql.execute("create database if not exists pseudo_col") + tdSql.execute('use pseudo_col') + tdSql.execute('create table st(ts timestamp, f int) tags (t int)') + tdSql.execute("insert into ct1 using st tags(1) values('2023-10-10 14:10:00', 1)('2023-10-10 14:10:01', 11)") + tdSql.execute("insert into ct2 using st tags(2) values('2023-10-10 14:10:02', 2)('2023-10-10 14:10:03', 22)") + + tdSql.query('select tbname from (select tbname from st) order by tbname') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + + tdSql.query('select `tbname` from (select tbname from st) order by tbname') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + + tdSql.query('select `st.tbname` from (select st.tbname from st) order by `st.tbname`') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + + tdSql.error('select tbname from (select * from st)') + tdSql.error('select st.tbname from (select st.tbname from st)') + tdSql.error('select `st.tbname` from (select st.tbname from st) order by tbname') + + tdSql.query('select _wstart, _wend, _wduration, c from (select _wstart, _wend, _wduration, count(*) as c from st interval(1s)) order by _wstart') + tdSql.checkCols(4) + tdSql.checkRows(4) + tdSql.checkData(0, 1, '2023-10-10 14:10:01') + tdSql.checkData(0, 3, 1) + + tdSql.error('select _wstart, _wend, _wduration, c from (select count(*) as c from st) order by _wstart') + + tdSql.query("select _irowts, if2 from (select _irowts, interp(f) as if2 from st range('2023-10-10 14:10:00', '2023-10-10 14:10:10') every(1s) fill(value, 8))") + tdSql.checkRows(11) + tdSql.checkData(9, 1, 8); + tdSql.execute('drop database pseudo_col') + + tdSql.error("select _irowts, if2 from (select interp(f) as if2 from st range('2023-10-10 14:10:00', '2023-10-10 14:10:10') every(1s) fill(value, 8))") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 36b8fded81..7afef03f0d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1263,6 +1263,7 @@ #develop test ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py +,,n,develop-test,python3 ./test.py -f 2-query/pseudo_column.py ,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py From 440fa772828229b99d21851e067d827a30f91381 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 16:55:36 +0800 Subject: [PATCH 024/177] fix:merge datablock if data in same wal version --- source/dnode/vnode/src/tq/tqRead.c | 82 ++++++++++-------------------- 1 file changed, 28 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 3b052a3edd..1c2561b1a8 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -366,82 +366,56 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con // todo ignore the error in wal? bool tqNextBlockInWal(STqReader* pReader, const char* id) { SWalReader* pWalReader = pReader->pWalReader; + SSDataBlock* pDataBlock = NULL; uint64_t st = taosGetTimestampMs(); while (1) { - SArray* pBlockList = pReader->submit.aSubmitTbData; - if (pBlockList == NULL || pReader->nextBlk >= taosArrayGetSize(pBlockList)) { - // try next message in wal file - // todo always retry to avoid read failure caused by wal file deletion - if (walNextValidMsg(pWalReader) < 0) { - return false; - } - - void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); - int64_t ver = pWalReader->pHead->head.version; - - SDecoder decoder = {0}; - tDecoderInit(&decoder, pBody, bodyLen); - - { - int32_t nSubmitTbData = taosArrayGetSize(pReader->submit.aSubmitTbData); - for (int32_t i = 0; i < nSubmitTbData; i++) { - SSubmitTbData* pData = taosArrayGet(pReader->submit.aSubmitTbData, i); - if (pData->pCreateTbReq != NULL) { - taosArrayDestroy(pData->pCreateTbReq->ctb.tagName); - taosMemoryFreeClear(pData->pCreateTbReq); - } - pData->aRowP = taosArrayDestroy(pData->aRowP); - } - pReader->submit.aSubmitTbData = taosArrayDestroy(pReader->submit.aSubmitTbData); - } - - if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { - tDecoderClear(&decoder); - tqError("decode wal file error, msgLen:%d, ver:%" PRId64, bodyLen, ver); - return false; - } - - tDecoderClear(&decoder); - pReader->nextBlk = 0; + // try next message in wal file + if (walNextValidMsg(pWalReader) < 0) { + return false; } + void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pWalReader->pHead->head.version; + + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); + pReader->nextBlk = 0; int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqTrace("tq reader next data block %d/%d, len:%d %" PRId64 " %d", pReader->nextBlk, - numOfBlocks, pReader->msg.msgLen, pReader->msg.ver, pReader->nextBlk); + tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, + numOfBlocks, pReader->msg.msgLen, pReader->msg.ver); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if (pReader->tbIdHash == NULL) { - SSDataBlock* pRes = NULL; - int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); - if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - return true; - } - } - - void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); - if (ret != NULL) { - tqTrace("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); - + if (pReader->tbIdHash == NULL || taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)) != NULL) { + tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid); SSDataBlock* pRes = NULL; int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - return true; + if(pDataBlock == NULL){ + pDataBlock = createOneDataBlock(pRes, true); + }else{ + blockDataMerge(pDataBlock, pRes); + } } } else { pReader->nextBlk += 1; tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); } } - - qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->msg.msgStr = NULL; + if(pDataBlock != NULL){ + blockDataCleanup(pReader->pResBlock); + copyDataBlock(pReader->pResBlock, pDataBlock); + blockDataDestroy(pDataBlock); + return true; + }else{ + qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); + } + if(taosGetTimestampMs() - st > 1000){ return false; } From e9fc079d269133f9e05dad8a60c04b2ed6c25e3e Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Tue, 10 Oct 2023 14:28:35 +0800 Subject: [PATCH 025/177] opt bloom filter --- include/util/tbloomfilter.h | 6 +++- include/util/tscalablebf.h | 3 ++ source/libs/stream/src/streamUpdate.c | 13 +++++--- source/util/src/tbloomfilter.c | 44 ++++++++++++++++----------- source/util/src/tscalablebf.c | 28 +++++++++++++++-- source/util/test/bloomFilterTest.cpp | 8 +++-- 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 16f7ff7958..23bf5aaafb 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -24,6 +24,9 @@ extern "C" { #endif +#define HASH_FUNCTION_1 taosFastHash +#define HASH_FUNCTION_2 taosDJB2Hash + typedef struct SBloomFilter { uint32_t hashFunctions; uint64_t expectedEntries; @@ -37,8 +40,9 @@ typedef struct SBloomFilter { } SBloomFilter; SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); +int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); void tBloomFilterDestroy(SBloomFilter *pBF); void tBloomFilterDump(const SBloomFilter *pBF); bool tBloomFilterIsFull(const SBloomFilter *pBF); diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 9977c1436d..2cf170cf04 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -26,9 +26,12 @@ typedef struct SScalableBf { SArray *bfArray; // array of bloom filters uint32_t growth; uint64_t numBits; + _hash_fn_t hashFn1; + _hash_fn_t hashFn2; } SScalableBf; SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); +int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); void tScalableBfDestroy(SScalableBf *pSBf); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index f9ab672c4b..a463a17ec8 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -218,17 +218,22 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } SScalableBf *pSBf = getSBf(pInfo, ts); - // pSBf may be a null pointer - if (pSBf) { - res = tScalableBfPut(pSBf, &updateKey, sizeof(SUpdateKey)); - } int32_t size = taosHashGetSize(pInfo->pMap); if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && *pMapMaxTs < ts)) { taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); + // pSBf may be a null pointer + if (pSBf) { + res = tScalableBfPutNoCheck(pSBf, &updateKey, sizeof(SUpdateKey)); + } return false; } + // pSBf may be a null pointer + if (pSBf) { + res = tScalableBfPut(pSBf, &updateKey, sizeof(SUpdateKey)); + } + if (!pMapMaxTs && maxTs < ts) { taosArraySet(pInfo->pTsBuckets, index, &ts); return false; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 84a78f3477..150faab571 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -24,9 +24,8 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; - buf[unitIndex] |= mask; + buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS)); return buf[unitIndex] != old; } @@ -57,10 +56,8 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { // ln(2) = 0.693147180559945 pBF->hashFunctions = (uint32_t)ceil(lnRate / 0.693147180559945); - /*pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);*/ - /*pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR);*/ - pBF->hashFn1 = taosFastHash; - pBF->hashFn2 = taosDJB2Hash; + pBF->hashFn1 = HASH_FUNCTION_1; + pBF->hashFn2 = HASH_FUNCTION_2; pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); @@ -69,14 +66,29 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { return pBF; } -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { +int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { ASSERT(!tBloomFilterIsFull(pBF)); + bool hasChange = false; + const register uint64_t size = pBF->numBits; + uint64_t cbHash = hash1; + for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { + hasChange |= setBit(pBF->buffer, cbHash % size); + cbHash += hash2; + } + if (hasChange) { + pBF->size++; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_FAILED; +} + +int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); bool hasChange = false; const register uint64_t size = pBF->numBits; uint64_t cbHash = h1; - for (uint64_t i = 0; i < pBF->hashFunctions; ++i) { + for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { hasChange |= setBit(pBF->buffer, cbHash % size); cbHash += h2; } @@ -87,16 +99,14 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, uint32_t len) { - uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); - uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); +int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { const register uint64_t size = pBF->numBits; - uint64_t cbHash = h1; - for (uint64_t i = 0; i < pBF->hashFunctions; ++i) { + uint64_t cbHash = hash1; + for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { if (!getBit(pBF->buffer, cbHash % size)) { return TSDB_CODE_SUCCESS; } - cbHash += h2; + cbHash += hash2; } return TSDB_CODE_FAILED; } @@ -137,10 +147,8 @@ SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) { if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; } if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; - /*pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);*/ - /*pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR);*/ - pBF->hashFn1 = taosFastHash; - pBF->hashFn2 = taosDJB2Hash; + pBF->hashFn1 = HASH_FUNCTION_1; + pBF->hashFn2 = HASH_FUNCTION_2; return pBF; _error: diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 797f3a924d..3b4975b701 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -39,13 +39,31 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { return NULL; } pSBf->growth = DEFAULT_GROWTH; + pSBf->hashFn1 = HASH_FUNCTION_1; + pSBf->hashFn2 = HASH_FUNCTION_2; return pSBf; } +int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + int32_t size = taosArrayGetSize(pSBf->bfArray); + SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + ASSERT(pNormalBf); + if (tBloomFilterIsFull(pNormalBf)) { + pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); + if (pNormalBf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return tBloomFilterPut(pNormalBf, keyBuf, len); +} + int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); + uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { - if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), keyBuf, len) != TSDB_CODE_SUCCESS) { + if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } } @@ -59,13 +77,15 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { return TSDB_CODE_OUT_OF_MEMORY; } } - return tBloomFilterPut(pNormalBf, keyBuf, len); + return tBloomFilterPutHash(pNormalBf, h1, h2); } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); + uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 1; i >= 0; --i) { - if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), keyBuf, len) != TSDB_CODE_SUCCESS) { + if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } } @@ -113,6 +133,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + pSBf->hashFn1 = HASH_FUNCTION_1; + pSBf->hashFn2 = HASH_FUNCTION_2; pSBf->bfArray = NULL; int32_t size = 0; if (tDecodeI32(pDecoder, &size) < 0) goto _error; diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 2e02129164..c51de3c8a4 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -43,12 +43,16 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_FAILED); + uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED); } for (int64_t i = 2000; i < 3000; i++) { int64_t ts = i + ts1; - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); + uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); } tBloomFilterDestroy(pBF1); From 71a3e3ce169290af7f676ab45282cef68431f257 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 11 Oct 2023 10:29:41 +0800 Subject: [PATCH 026/177] ci --- tests/script/tsim/stream/checkpointSession1.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/stream/checkpointSession1.sim b/tests/script/tsim/stream/checkpointSession1.sim index 5c9625aabb..e0ba08f9e7 100644 --- a/tests/script/tsim/stream/checkpointSession1.sim +++ b/tests/script/tsim/stream/checkpointSession1.sim @@ -63,7 +63,7 @@ sql insert into t2 values(1648791233003,4,2,3,1.1); $loop_count = 0 loop1: -sleep 1000 +sleep 2000 sql select * from streamt; From 8362759e471d32870ef74907209bb8bd241a94c1 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 11 Oct 2023 13:46:11 +0800 Subject: [PATCH 027/177] ci --- tests/script/tsim/stream/checkpointSession1.sim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/stream/checkpointSession1.sim b/tests/script/tsim/stream/checkpointSession1.sim index e0ba08f9e7..ea93e1525a 100644 --- a/tests/script/tsim/stream/checkpointSession1.sim +++ b/tests/script/tsim/stream/checkpointSession1.sim @@ -57,13 +57,15 @@ system sh/stop_dnodes.sh system sh/exec.sh -n dnode1 -s start +sleep 2000 + sql insert into t1 values(1648791213002,3,2,3,1.1); sql insert into t2 values(1648791233003,4,2,3,1.1); $loop_count = 0 loop1: -sleep 2000 +sleep 1000 sql select * from streamt; From 8253874398cc78fa2d76df35963e0e290154c8a4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 11 Oct 2023 14:47:26 +0800 Subject: [PATCH 028/177] enh(tsdb/cos): new sdk for s3 --- cmake/cmake.options | 18 ++ cmake/libs3.GNUmakefile | 430 ++++++++++++++++++++++++++++++ cmake/libs3_CMakeLists.txt.in | 28 ++ cmake/ssl_CMakeLists.txt.in | 15 ++ contrib/CMakeLists.txt | 54 ++++ source/dnode/vnode/CMakeLists.txt | 7 +- 6 files changed, 551 insertions(+), 1 deletion(-) create mode 100644 cmake/libs3.GNUmakefile create mode 100644 cmake/libs3_CMakeLists.txt.in create mode 100644 cmake/ssl_CMakeLists.txt.in diff --git a/cmake/cmake.options b/cmake/cmake.options index 1d4e9ba515..bacc5adfb7 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -127,6 +127,22 @@ option( IF(${TD_LINUX}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) + +IF(${BUILD_WITH_S3}) + +option( + BUILD_WITH_COS + "If build with cos" + OFF +) + +ELSE () + option( BUILD_WITH_COS "If build with cos" @@ -135,6 +151,8 @@ option( ENDIF () +ENDIF () + option( BUILD_WITH_SQLITE "If build with sqlite" diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile new file mode 100644 index 0000000000..abf954f5c1 --- /dev/null +++ b/cmake/libs3.GNUmakefile @@ -0,0 +1,430 @@ +# GNUmakefile +# +# Copyright 2008 Bryan Ischo +# +# This file is part of libs3. +# +# libs3 is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. +# +# In addition, as a special exception, the copyright holders give +# permission to link the code of this library and its programs with the +# OpenSSL library, and distribute linked combinations including the two. +# +# libs3 is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# version 3 along with libs3, in a file named COPYING. If not, see +# . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . + +# I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools +# but I just couldn't stomach them. Since this is a Makefile for POSIX +# systems, I will simply do away with autohell completely and use a GNU +# Makefile. GNU make ought to be available pretty much everywhere, so I +# don't see this being a significant issue for portability. + +# All commands assume a GNU compiler. For systems which do not use a GNU +# compiler, write scripts with the same names as these commands, and taking +# the same arguments, and translate the arguments and commands into the +# appropriate non-POSIX ones as needed. libs3 assumes a GNU toolchain as +# the most portable way to build software possible. Non-POSIX, non-GNU +# systems can do the work of supporting this build infrastructure. + + +# -------------------------------------------------------------------------- +# Set libs3 version number, unless it is already set. + +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 +LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) + + +# ----------------------------------------------------------------------------- +# Determine verbosity. VERBOSE_SHOW should be prepended to every command which +# should only be displayed if VERBOSE is set. QUIET_ECHO may be used to +# echo text only if VERBOSE is not set. Typically, a VERBOSE_SHOW command will +# be paired with a QUIET_ECHO command, to provide a command which is displayed +# in VERBOSE mode, along with text which is displayed in non-VERBOSE mode to +# describe the command. +# +# No matter what VERBOSE is defined to, it ends up as true if it's defined. +# This will be weird if you defined VERBOSE=false in the environment, and we +# switch it to true here; but the meaning of VERBOSE is, "if it's defined to +# any value, then verbosity is turned on". So don't define VERBOSE if you +# don't want verbosity in the build process. +# ----------------------------------------------------------------------------- + +ifdef VERBOSE + VERBOSE = true + VERBOSE_ECHO = @ echo + VERBOSE_SHOW = + QUIET_ECHO = @ echo > /dev/null +else + VERBOSE = false + VERBOSE_ECHO = @ echo > /dev/null + VERBOSE_SHOW = @ + QUIET_ECHO = @ echo +endif + + +# -------------------------------------------------------------------------- +# BUILD directory +ifndef BUILD + ifdef DEBUG + BUILD := build-debug + else + BUILD := build + endif +endif + + +# -------------------------------------------------------------------------- +# DESTDIR directory +ifndef DESTDIR + DESTDIR := ${HOME}/.cos-local.1 +endif + +# -------------------------------------------------------------------------- +# LIBDIR directory +ifndef LIBDIR + LIBDIR := ${DESTDIR}/lib +endif + +# -------------------------------------------------------------------------- +# Compiler CC handling +ifndef CC + CC := gcc +endif + +# -------------------------------------------------------------------------- +# Acquire configuration information for libraries that libs3 depends upon + +ifndef CURL_LIBS + CURL_LIBS := $(shell curl-config --libs) +endif + +ifndef CURL_CFLAGS + CURL_CFLAGS := $(shell curl-config --cflags) +endif + +ifndef LIBXML2_LIBS + LIBXML2_LIBS := $(shell xml2-config --libs) +endif + +ifndef LIBXML2_CFLAGS + LIBXML2_CFLAGS := $(shell xml2-config --cflags) +endif + +ifndef OPENSSL_LIBS + OPENSSL_LIBS := -lssl -lcrypto +endif + +# -------------------------------------------------------------------------- +# These CFLAGS assume a GNU compiler. For other compilers, write a script +# which converts these arguments into their equivalent for that particular +# compiler. + +ifndef CFLAGS + ifdef DEBUG + CFLAGS := -g + else + CFLAGS := -O3 + endif +endif + +CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ + $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ + -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ + -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ + -DLIBS3_VER=\"$(LIBS3_VER)\" \ + -D__STRICT_ANSI__ \ + -D_ISOC99_SOURCE \ + -D_POSIX_C_SOURCE=200112L + +LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread + +STRIP ?= strip +INSTALL := install --strip-program=$(STRIP) + + +# -------------------------------------------------------------------------- +# Default targets are everything + +.PHONY: all +all: exported test + + +# -------------------------------------------------------------------------- +# Exported targets are the library and driver program + +.PHONY: exported +exported: libs3 s3 headers +exported_static: $(LIBS3_STATIC) + +# -------------------------------------------------------------------------- +# Install target + +.PHONY: install install_static +install_static: exported_static + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r inc/libs3.h \ + $(DESTDIR)/include/libs3.h + +install: exported + $(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ + $(DESTDIR)/bin/s3 + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \ + $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER) + $(QUIET_ECHO) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR): Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER) \ + $(LIBDIR)/libs3.so.$(LIBS3_VER_MAJOR) + $(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library + $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so + $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(LIBDIR)/libs3.a + $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ + $(DESTDIR)/include/libs3.h + + +# -------------------------------------------------------------------------- +# Uninstall target + +.PHONY: uninstall +uninstall: + $(QUIET_ECHO) Installed files: Uninstalling + $(VERBOSE_SHOW) \ + rm -f $(DESTDIR)/bin/s3 \ + $(DESTDIR)/include/libs3.h \ + $(DESTDIR)/lib/libs3.a \ + $(DESTDIR)/lib/libs3.so \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ + $(DESTDIR)/lib/libs3.so.$(LIBS3_VER) + + +# -------------------------------------------------------------------------- +# Compile target patterns + +$(BUILD)/obj/%.o: src/%.c + $(QUIET_ECHO) $@: Compiling object + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.d) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -o $@ -c $< + +$(BUILD)/obj/%.do: src/%.c + $(QUIET_ECHO) $@: Compiling dynamic object + $(QUIET_ECHO) cflags:${CFLAGS} + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< + + +# -------------------------------------------------------------------------- +# libs3 library targets + +LIBS3_SHARED = $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) +LIBS3_STATIC = $(BUILD)/lib/libs3.a + +.PHONY: libs3 +libs3: $(LIBS3_SHARED) $(LIBS3_STATIC) + +LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \ + object.c request.c request_context.c \ + response_headers_handler.c service_access_logging.c \ + service.c simplexml.c util.c multipart.c + +$(LIBS3_SHARED): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.do) + $(QUIET_ECHO) $@: Building shared library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -shared -Wl,-soname,libs3.so.$(LIBS3_VER_MAJOR) \ + -o $@ $^ $(LDFLAGS) + +$(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o) + $(QUIET_ECHO) $@: Building static library + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(AR) cr $@ $^ + + +# -------------------------------------------------------------------------- +# Driver program targets + +.PHONY: s3 +s3: $(BUILD)/bin/s3 + +$(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) + + +# -------------------------------------------------------------------------- +# libs3 header targets + +.PHONY: headers +headers: $(BUILD)/include/libs3.h + +$(BUILD)/include/libs3.h: inc/libs3.h + $(QUIET_ECHO) $@: Linking header + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) ln -sf $(abspath $<) $@ + + +# -------------------------------------------------------------------------- +# Test targets + +.PHONY: test +test: $(BUILD)/bin/testsimplexml + +$(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS) + + +# -------------------------------------------------------------------------- +# Clean target + +.PHONY: clean +clean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + +.PHONY: distclean +distclean: + $(QUIET_ECHO) $(BUILD): Cleaning + $(VERBOSE_SHOW) rm -rf $(BUILD) + + +# -------------------------------------------------------------------------- +# Clean dependencies target + +.PHONY: cleandeps +cleandeps: + $(QUIET_ECHO) $(BUILD)/dep: Cleaning dependencies + $(VERBOSE_SHOW) rm -rf $(BUILD)/dep + + +# -------------------------------------------------------------------------- +# Dependencies + +ALL_SOURCES := $(LIBS3_SOURCES) s3.c testsimplexml.c + +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.d))) +$(foreach i, $(ALL_SOURCES), $(eval -include $(BUILD)/dep/src/$(i:%.c=%.dd))) + + +# -------------------------------------------------------------------------- +# Debian package target + +DEBPKG = $(BUILD)/pkg/libs3_$(LIBS3_VER).deb +DEBDEVPKG = $(BUILD)/pkg/libs3-dev_$(LIBS3_VER).deb + +.PHONY: deb +deb: $(DEBPKG) $(DEBDEVPKG) + +$(DEBPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBPKG): exported $(BUILD)/deb/DEBIAN/control $(BUILD)/deb/DEBIAN/shlibs \ + $(BUILD)/deb/DEBIAN/postinst \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.gz \ + $(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz \ + $(BUILD)/deb/usr/share/doc/libs3/copyright + DESTDIR=$(BUILD)/deb/usr $(MAKE) install + rm -rf $(BUILD)/deb/usr/include + rm -f $(BUILD)/deb/usr/lib/libs3.a + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb $@ + mv $@ $(BUILD)/pkg/libs3_$(LIBS3_VER)_$(DEBARCH).deb + +$(DEBDEVPKG): DEBARCH = $(shell dpkg-architecture | grep ^DEB_BUILD_ARCH= | \ + cut -d '=' -f 2) +$(DEBDEVPKG): exported $(BUILD)/deb-dev/DEBIAN/control \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz \ + $(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright + DESTDIR=$(BUILD)/deb-dev/usr $(MAKE) install + rm -rf $(BUILD)/deb-dev/usr/bin + rm -f $(BUILD)/deb-dev/usr/lib/libs3.so* + @mkdir -p $(dir $@) + fakeroot dpkg-deb -b $(BUILD)/deb-dev $@ + mv $@ $(BUILD)/pkg/libs3-dev_$(LIBS3_VER)_$(DEBARCH).deb + +$(BUILD)/deb/DEBIAN/control: debian/control + @mkdir -p $(dir $@) + echo -n "Depends: " > $@ + dpkg-shlibdeps -Sbuild -O $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) | \ + cut -d '=' -f 2- >> $@ + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' | \ + grep -v ^Source: >> $@ + +$(BUILD)/deb-dev/DEBIAN/control: debian/control.dev + @mkdir -p $(dir $@) + sed -e 's/LIBS3_VERSION/$(LIBS3_VER)/' \ + < $< | sed -e 's/DEBIAN_ARCHITECTURE/$(DEBARCH)/' > $@ + +$(BUILD)/deb/DEBIAN/shlibs: + echo -n "libs3 $(LIBS3_VER_MAJOR) libs3 " > $@ + echo "(>= $(LIBS3_VER))" >> $@ + +$(BUILD)/deb/DEBIAN/postinst: debian/postinst + @mkdir -p $(dir $@) + cp $< $@ + +$(BUILD)/deb/usr/share/doc/libs3/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/copyright: LICENSE + @mkdir -p $(dir $@) + cp $< $@ + @echo >> $@ + @echo -n "An alternate location for the GNU General Public " >> $@ + @echo "License version 3 on Debian" >> $@ + @echo "systems is /usr/share/common-licenses/GPL-3." >> $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.gz: debian/changelog + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb/usr/share/doc/libs3/changelog.Debian.gz: debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + +$(BUILD)/deb-dev/usr/share/doc/libs3-dev/changelog.Debian.gz: \ + debian/changelog.Debian + @mkdir -p $(dir $@) + gzip --best -c $< > $@ + + diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in new file mode 100644 index 0000000000..b77299baf3 --- /dev/null +++ b/cmake/libs3_CMakeLists.txt.in @@ -0,0 +1,28 @@ +# libs3 + +set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ") + +set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /") + +function(update_cflags) + file(READ "GNUmakefile" S3_CONTENT) + string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}") + file(WRITE "GNUmakefile" "${S3_CONTENT}") +endfunction(update_cflags) + +ExternalProject_Add(libs3 + GIT_REPOSITORY https://github.com/bji/libs3 + #GIT_TAG v5.0.16 + SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + BUILD_ALWAYS 1 + UPDATE_COMMAND "" + #sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /" + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile + #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a + #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static + TEST_COMMAND "" +) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in new file mode 100644 index 0000000000..7821848618 --- /dev/null +++ b/cmake/ssl_CMakeLists.txt.in @@ -0,0 +1,15 @@ +# openssl +ExternalProject_Add(openssl + URL https://www.openssl.org/source/openssl-3.1.3.tar.gz + URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/openssl" + BUILD_IN_SOURCE TRUE + #BUILD_ALWAYS 1 + #UPDATE_COMMAND "" + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared + BUILD_COMMAND make -j + INSTALL_COMMAND make install + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e3e48ac3a1..39bf85df8f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,6 +8,37 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) + if(${BUILD_WITH_S3}) + +set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") +configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) + +if(${BUILD_WITH_COS}) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) +endif(${BUILD_WITH_COS}) + +configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") +execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") + +set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") +configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + +if(${BUILD_WITH_COS}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) +endif(${BUILD_WITH_COS}) + +configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") +execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") + +else() + set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -37,6 +68,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") +endif(${BUILD_WITH_S3}) + endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") @@ -155,6 +188,18 @@ if(${BUILD_WITH_SQLITE}) cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_SQLITE}) +# s3 +if(${BUILD_WITH_S3}) + #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_S3) + +else() + # cos if(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -165,6 +210,8 @@ if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene if(${BUILD_WITH_LUCENE}) cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -390,6 +437,11 @@ if (${BUILD_WITH_ROCKSDB}) endif() endif() +if(${BUILD_WITH_S3}) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + +else() + # cos if(${BUILD_WITH_COS}) if(${TD_LINUX}) @@ -415,6 +467,8 @@ if(${BUILD_WITH_COS}) endif(${TD_LINUX}) endif(${BUILD_WITH_COS}) +endif(${BUILD_WITH_S3}) + # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 84d54f3350..516c8a8b69 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,6 +162,12 @@ target_link_libraries( ) if(${TD_LINUX}) + if(${BUILD_WITH_S3}) + + endif(${BUILD_WITH_S3}) + +if(${BUILD_WITH_COS}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) @@ -194,7 +200,6 @@ target_include_directories( PUBLIC "$ENV{HOME}/.cos-local.1/include" ) -if(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) From 5fafdf351b0cde9080338514ddac15014708b6d5 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 11 Oct 2023 20:17:47 +0800 Subject: [PATCH 029/177] refect --- source/libs/executor/src/streamtimewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9da3c99e7..6fc862b438 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3590,7 +3590,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) { for (int32_t i = 0; i < num; i++) { SStateWindowInfo curInfo = {0}; SStateWindowInfo nextInfo = {0}; - SStateWindowInfo dummy = {0}; qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo); From 1e1ed541fa83d900189e4c20b198c83eaf3e336c Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 13 Oct 2023 08:11:15 +0800 Subject: [PATCH 030/177] fix: add more test case --- tests/develop-test/2-query/pseudo_column.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/develop-test/2-query/pseudo_column.py b/tests/develop-test/2-query/pseudo_column.py index 9f6366a3c6..4a97be6612 100644 --- a/tests/develop-test/2-query/pseudo_column.py +++ b/tests/develop-test/2-query/pseudo_column.py @@ -47,6 +47,14 @@ class TDTestCase: tdSql.checkData(2, 0, 'ct2') tdSql.checkData(2, 0, 'ct2') + tdSql.query('select `tbname` from (select tbname from st) order by tbname') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + tdSql.query('select `st.tbname` from (select st.tbname from st) order by `st.tbname`') tdSql.checkCols(1) tdSql.checkRows(4) From 429125be933cb6297680ec3fc07a25d50cf460d8 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 13 Oct 2023 11:32:35 +0800 Subject: [PATCH 031/177] session state recover --- include/libs/stream/tstreamFileState.h | 3 +++ .../executor/src/streamtimewindowoperator.c | 1 - source/libs/stream/src/tstreamFileState.c | 26 ++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 2b567a7370..c1974df7de 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -79,6 +79,9 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen); int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); +SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); +int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); + void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateCleanup(void* pBuff); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9da3c99e7..6fc862b438 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3590,7 +3590,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) { for (int32_t i = 0; i < num; i++) { SStateWindowInfo curInfo = {0}; SStateWindowInfo nextInfo = {0}; - SStateWindowInfo dummy = {0}; qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index ac404893f0..584e81fafc 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -177,7 +177,10 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ // todo(liuyao) optimize if (type == STREAM_STATE_BUFF_HASH) { recoverSnapshot(pFileState, checkpointId); + } else { + recoverSesssion(pFileState, checkpointId); } + return pFileState; _error: @@ -642,12 +645,24 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { } int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { - int code = TSDB_CODE_SUCCESS; + int code = TSDB_CODE_SUCCESS; + if (pFileState->maxTs != INT64_MIN) { + int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) + ? INT64_MIN + : pFileState->maxTs - pFileState->deleteMark; + deleteExpiredCheckPoint(pFileState, mark); + } + SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore); if (pCur == NULL) { return -1; } + int32_t recoverNum = TMIN(MIN_NUM_OF_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { + if (pFileState->curRowCount >= recoverNum) { + break; + } + void* pVal = NULL; int32_t vlen = 0; SSessionKey key = {0}; @@ -655,12 +670,14 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { if (code != 0) { break; } - taosMemoryFree(pVal); + SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); + putSessionWinResultBuff(pFileState, pPos); code = streamStateSessionCurPrev_rocksdb(pCur); } streamStateFreeCur(pCur); return code; } + int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { @@ -674,11 +691,12 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { if (pCur == NULL) { return -1; } - + int32_t recoverNum = TMIN(MIN_NUM_OF_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { - if (pFileState->curRowCount == pFileState->maxRowCount) { + if (pFileState->curRowCount >= recoverNum) { break; } + void* pVal = NULL; int32_t vlen = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); From 0e31231212417690974c3d641b5b29ffa25a34d8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 13 Oct 2023 16:22:32 +0800 Subject: [PATCH 032/177] test: support replica3 in splited/transform tmqvnode --- .../5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 4 +- .../system-test/7-tmq/tmqVnodeSplit-column.py | 213 +++++++++ tests/system-test/7-tmq/tmqVnodeSplit-db.py | 214 +++++++++ ...tmqVnodeSplit-stb-select-duplicatedata.py} | 23 +- .../7-tmq/tmqVnodeSplit-stb-select.py | 213 +++++++++ tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 213 +++++++++ .../7-tmq/tmqVnodeTransform-stb.py | 433 ++++++++++++++++++ 7 files changed, 1301 insertions(+), 12 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-column.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-db.py rename tests/system-test/7-tmq/{tmqVnodeSplit.py => tmqVnodeSplit-stb-select-duplicatedata.py} (93%) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-stb.py create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-stb.py diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 3a972ff4e9..fb62110b14 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -146,9 +146,9 @@ class TDTestCase: # a11111=paraDict["dbNumbers"] # print(f"==================={dbNameIndex},{a11111}") threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) - + newTdSql2=tdCom.newTdSql() redbNameIndex = '%s%d'%(paraDict["dbName"],i+100) - threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql2, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) for tr in threads: tr.start() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py new file mode 100644 index 0000000000..95c35363a8 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 >= 0 "%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py new file mode 100644 index 0000000000..1c2e867758 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -0,0 +1,214 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from db") + queryString = "database %s"%(paraDict['dbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py similarity index 93% rename from tests/system-test/7-tmq/tmqVnodeSplit.py rename to tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 67deb84620..086fe6e4c5 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -58,7 +58,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdCom.drop_all_db(); + tdCom.drop_all_db() tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") @@ -112,13 +112,13 @@ class TDTestCase: 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', + 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -157,7 +157,13 @@ class TDTestCase: tdLog.info("create ctb1") tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) tmqCom.getStartConsumeNotifyFromTmqsim() @@ -169,11 +175,8 @@ class TDTestCase: # split vgroup self.splitVgroups() - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") + + tdLog.info("insert ctb2 data") pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) pInsertThread.join() pInsertThread1.join() @@ -181,7 +184,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 >= resultList[0] or expectrowcnt <= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py new file mode 100644 index 0000000000..3214c2f5c4 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 120, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0] or expectrowcnt < resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py new file mode 100644 index 0000000000..27d296ed0e --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -0,0 +1,213 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb ") + queryString = "stable %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(True) + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py new file mode 100644 index 0000000000..18f9fafe9a --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -0,0 +1,433 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCaseStable(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 >= resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCaseNtable(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCaseStableSelect(self): + tdLog.printNoPrefix("======== test case subscrib column start : ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic3'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(5) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case subscrib column end ...... ") + + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(5) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(5) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case subscrib Dbname end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCaseStable() + self.prepareTestEnv() + self.tmqCaseNtable() + self.prepareTestEnv() + self.tmqCaseStableSelect() + self.prepareTestEnv() + self.tmqCaseDbname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 0a1ff3283e50db9bde8cfe62a8a99166bd04504d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 16:59:05 +0800 Subject: [PATCH 033/177] vnode/cos: first round size delte and get --- cmake/curl_CMakeLists.txt.in | 3 +- cmake/libs3_CMakeLists.txt.in | 1 + contrib/CMakeLists.txt | 14 +- source/common/src/tglobal.c | 16 +- source/dnode/vnode/CMakeLists.txt | 29 +++- source/dnode/vnode/src/vnd/vnodeCos.c | 205 +++++++++++++++++++++++++- 6 files changed, 251 insertions(+), 17 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 1f2291c519..24a1210422 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -10,7 +10,8 @@ ExternalProject_Add(curl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd #CONFIGURE_COMMAND ./configure --without-ssl BUILD_COMMAND make INSTALL_COMMAND make install diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b77299baf3..b54a084f66 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,6 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 + DEPENDS curl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 39bf85df8f..1588eb4025 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -13,10 +13,8 @@ if(${TD_LINUX}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) -if(${BUILD_WITH_COS}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) -endif(${BUILD_WITH_COS}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -27,9 +25,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) -if(${BUILD_WITH_COS}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) -endif(${BUILD_WITH_COS}) configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -129,6 +125,9 @@ if(${BUILD_TEST}) cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) +# xml2 +cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + # lz4 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -195,6 +194,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) @@ -295,6 +295,9 @@ target_include_directories( ) unset(CMAKE_PROJECT_INCLUDE_BEFORE) +# xml2 +add_subdirectory(xml2 EXCLUDE_FROM_ALL) + # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( @@ -439,6 +442,7 @@ endif() if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + MESSAGE("build with s3: ${BUILD_WITH_S3}") else() @@ -449,7 +453,7 @@ if(${BUILD_WITH_COS}) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) option(ENABLE_TEST "Enable the tests" OFF) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - MESSAGE("$ENV{HOME}/.cos-local.1/include") + #MESSAGE("$ENV{HOME}/.cos-local.1/include") set(CMAKE_BUILD_TYPE debug) set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME}) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 92b5ff2828..8ea5528fbb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,12 +15,12 @@ #define _DEFAULT_SOURCE #include "tglobal.h" +#include "defines.h" #include "os.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" #include "tmisce.h" -#include "defines.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -221,7 +221,7 @@ float tsFPrecision = 1E-8; // float column precision double tsDPrecision = 1E-16; // double column precision uint32_t tsMaxRange = 500; // max quantization intervals uint32_t tsCurRange = 100; // current quantization intervals -bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom +bool tsIfAdtFse = false; // ADT-FSE algorithom or original huffman algorithom char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR // udf @@ -266,6 +266,9 @@ char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_FQDN_LEN] = ""; int8_t tsS3Enabled = false; +int8_t tsS3Https = true; +char tsS3Hostname[TSDB_FQDN_LEN] = ""; + int32_t tsS3BlockSize = 4096; // number of tsdb pages int32_t tsS3BlockCacheSize = 16; // number of blocks @@ -307,6 +310,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN); tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN); tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN); + char *proto = strstr(tsS3Endpoint, "https://"); + if (!proto) { + tsS3Https = false; + tstrncpy(tsS3Hostname, tsS3Endpoint + 7, TSDB_FQDN_LEN); + } else { + tstrncpy(tsS3Hostname, tsS3Endpoint + 8, TSDB_FQDN_LEN); + } + char *cos = strstr(tsS3Endpoint, "cos."); if (cos) { char *appid = strrchr(tsS3BucketName, '-'); @@ -1086,7 +1097,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval; tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor)); - tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 516c8a8b69..e554d94e4d 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -162,9 +162,34 @@ target_link_libraries( ) if(${TD_LINUX}) - if(${BUILD_WITH_S3}) - endif(${BUILD_WITH_S3}) +if(${BUILD_WITH_S3}) + MESSAGE("build with s3: ${BUILD_WITH_S3}") + target_include_directories( + vnode + + PUBLIC "$ENV{HOME}/.cos-local.1/include" + ) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) + find_library(S3_LIBRARY s3) + find_library(CURL_LIBRARY curl) + find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64) + find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64) + target_link_libraries( + vnode + + # s3 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 + ) + + add_definitions(-DUSE_S3) +endif(${BUILD_WITH_S3}) if(${BUILD_WITH_COS}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 5f650be97d..8984d33ad8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -2,13 +2,206 @@ #include "vndCos.h" -extern char tsS3Endpoint[]; -extern char tsS3AccessKeyId[]; -extern char tsS3AccessKeySecret[]; -extern char tsS3BucketName[]; -extern char tsS3AppId[]; +extern char tsS3Endpoint[]; +extern char tsS3AccessKeyId[]; +extern char tsS3AccessKeySecret[]; +extern char tsS3BucketName[]; +extern char tsS3AppId[]; +extern char tsS3Hostname[]; +extern int8_t tsS3Https; + +#if defined(USE_S3) + +#include "libs3.h" + +static int verifyPeerG = 0; +static const char *awsRegionG = NULL; +static int forceG = 0; +static int showResponsePropertiesG = 0; +static S3Protocol protocolG = S3ProtocolHTTPS; +// static S3Protocol protocolG = S3ProtocolHTTP; +static S3UriStyle uriStyleG = S3UriStylePath; +static int retriesG = 5; +static int timeoutMsG = 0; + +static int32_t s3Begin() { + S3Status status; + const char *hostname = tsS3Hostname; + const char *env_hn = getenv("S3_HOSTNAME"); + + if (env_hn) { + hostname = env_hn; + } + + if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { + vError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); + return -1; + } + + protocolG = !tsS3Https; + + return 0; +} + +static void s3End() { S3_deinitialize(); } +int32_t s3Init() { return s3Begin(); } + +void s3CleanUp() { s3End(); } + +static int should_retry() { + /* + if (retriesG--) { + // Sleep before next retry; start out with a 1 second sleep + static int retrySleepInterval = 1 * SLEEP_UNITS_PER_SECOND; + sleep(retrySleepInterval); + // Next sleep 1 second longer + retrySleepInterval++; + return 1; + } + */ + + return 0; +} + +typedef struct { + uint64_t content_length; + int status; + char *buf; + char err_msg[4096]; +} TS3SizeCBD; + +static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { + //(void)callbackData; + TS3SizeCBD *cbd = callbackData; + if (properties->contentLength > 0) { + cbd->content_length = properties->contentLength; + } else { + cbd->content_length = 0; + } + + return S3StatusOK; +} + +static void responseCompleteCallback(S3Status status, const S3ErrorDetails *error, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + cbd->status = status; + + int len = 0; + const int elen = sizeof(cbd->err_msg); + if (error) { + if (error->message) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); + } + if (error->resource) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); + } + if (error->furtherDetails) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); + } + if (error->extraDetailsCount) { + len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); + for (int i = 0; i < error->extraDetailsCount; i++) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, + error->extraDetails[i].value); + } + } + } +} + +int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } +void s3DeleteObjectsByPrefix(const char *prefix) {} + +void s3DeleteObjects(const char *object_name[], int nobject) { + int status = 0; + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; + + for (int i = 0; i < nobject; ++i) { + TS3SizeCBD cbd = {0}; + do { + S3_delete_object(&bucketContext, object_name[i], 0, timeoutMsG, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + } +} + +static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) { + TS3SizeCBD *cbd = callbackData; + if (cbd->content_length != bufferSize) { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } + + char *buf = taosMemoryCalloc(1, bufferSize); + if (buf) { + memcpy(buf, buffer, bufferSize); + + cbd->status = S3StatusOK; + return S3StatusOK; + } else { + cbd->status = S3StatusAbortedByCallback; + return S3StatusAbortedByCallback; + } +} + +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { + int status = 0; + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; + const char *ifMatch = 0, *ifNotMatch = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; + S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &getObjectDataCallback}; + + TS3SizeCBD cbd = {0}; + cbd.content_length = size; + do { + S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if (cbd.status != S3StatusOK) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + return TAOS_SYSTEM_ERROR(EIO); + } + + *ppBlock = cbd.buf; + + return 0; +} + +long s3Size(const char *object_name) { + long size = 0; + int status = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ResponseHandler responseHandler = {&responsePropertiesCallback, &responseCompleteCallback}; + + TS3SizeCBD cbd = {0}; + do { + S3_head_object(&bucketContext, object_name, 0, 0, &responseHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { + vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + } + + size = cbd.content_length; + + return size; +} + +void s3EvictCache(const char *path, long object_size) {} + +#elif defined(USE_COS) -#ifdef USE_COS #include "cos_api.h" #include "cos_http_io.h" #include "cos_log.h" From 11f7cdcf9d3b6faea3db55cbc4227ecbee29b4cc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:16:19 +0800 Subject: [PATCH 034/177] cos/xml2: cmake file for xml2 --- cmake/xml2_CMakeLists.txt.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cmake/xml2_CMakeLists.txt.in diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in new file mode 100644 index 0000000000..4c234d7a37 --- /dev/null +++ b/cmake/xml2_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# xml2 +ExternalProject_Add(xml2 + GIT_REPOSITORY https://github.com/GNOME/libxml2 + GIT_TAG v2.11.5 + SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) From c538478d0f91c26c34f998a31b0461016a769e9a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:30:09 +0800 Subject: [PATCH 035/177] contrib/cmake: fix curl file --- contrib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1588eb4025..1291506136 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -194,7 +194,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) From f66e57bf1f650139e5d40ba939387da72edd7d2c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 13 Oct 2023 17:47:42 +0800 Subject: [PATCH 036/177] contrib/cos: make libs3 depends xml2 --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b54a084f66..93d0eff41a 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl + DEPENDS curl xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1291506136..24ad63db13 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -125,9 +125,6 @@ if(${BUILD_TEST}) cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) -# xml2 -cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - # lz4 cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -194,6 +191,8 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + + cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) From d30c6f6bcc1e6a1102c57178a389b47b2976693c Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 16 Oct 2023 13:56:15 +0800 Subject: [PATCH 037/177] fix: change tablename.tbname aliasname to tbname --- source/libs/parser/src/parAstCreater.c | 8 ++- source/libs/parser/src/parTranslater.c | 57 +++++++++++++-------- tests/develop-test/2-query/pseudo_column.py | 10 +++- 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a418dc63b4..12062e0d4a 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -276,11 +276,9 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName); strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else if (pRawExpr->isPseudoColumn) { - int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->userAlias, pRawExpr->p, len); - pExpr->userAlias[len] = '\0'; - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + // all pseudo column are translate to function with same name + strcpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName); + strcpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 090d83d88f..42ab2f7a2f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1942,6 +1942,22 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_OUT_OF_MEMORY; } SExprNode* pOldExpr = (SExprNode*)(*ppNode); + //rewrite a.tbname == tbname(a) + if (nodeType(*ppNode) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(*ppNode))->funcType == FUNCTION_TYPE_TBNAME) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (0 != LIST_LENGTH(pFunc->pParameterList)) { + SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + pCol->node.resType = pOldExpr->resType; + strcpy(pCol->tableAlias, pVal->literal); + strcpy(pCol->colName, pFunc->functionName); + strcpy(pCol->node.aliasName, pCol->colName); + strcpy(pCol->node.userAlias, pCol->colName); + nodesDestroyNode(*ppNode); + *ppNode = (SNode*)pCol; + + return TSDB_CODE_SUCCESS; + } + } pCol->node.resType = pOldExpr->resType; strcpy(pCol->node.aliasName, pOldExpr->aliasName); strcpy(pCol->node.userAlias, pOldExpr->userAlias); @@ -1953,6 +1969,19 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_SUCCESS; } +static int32_t rewriteToColumnAndRetranslate(STranslateContext* pCxt, SNode** ppNode, int32_t errCode) { + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, errCode); + } else { + return TSDB_CODE_SUCCESS; + } +} + static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { @@ -1967,16 +1996,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** } if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { *pRewriteToColumn = true; - int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - translateColumn(pCxt, (SColumnNode**)ppNode); - if (pCxt->errCode != TSDB_CODE_SUCCESS) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); - } else { - return TSDB_CODE_SUCCESS; - } + return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_WINDOW_PC); } return TSDB_CODE_SUCCESS; } @@ -1992,23 +2012,18 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp } if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { *pRewriteToColumn = true; - int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - translateColumn(pCxt, (SColumnNode**)ppNode); - if (pCxt->errCode != TSDB_CODE_SUCCESS) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } else { - return TSDB_CODE_SUCCESS; - } + return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_TBNAME); } } else { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); STableNode* pTable = NULL; pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); - if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } + if (nodeType(pTable) != QUERY_NODE_REAL_TABLE) { + *pRewriteToColumn = true; + return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_TBNAME); } } return TSDB_CODE_SUCCESS; diff --git a/tests/develop-test/2-query/pseudo_column.py b/tests/develop-test/2-query/pseudo_column.py index 4a97be6612..1d94df4cff 100644 --- a/tests/develop-test/2-query/pseudo_column.py +++ b/tests/develop-test/2-query/pseudo_column.py @@ -55,7 +55,7 @@ class TDTestCase: tdSql.checkData(2, 0, 'ct2') tdSql.checkData(2, 0, 'ct2') - tdSql.query('select `st.tbname` from (select st.tbname from st) order by `st.tbname`') + tdSql.query('select tbname from (select st.tbname from st) order by tbname') tdSql.checkCols(1) tdSql.checkRows(4) tdSql.checkData(0, 0, 'ct1') @@ -63,6 +63,14 @@ class TDTestCase: tdSql.checkData(2, 0, 'ct2') tdSql.checkData(2, 0, 'ct2') + tdSql.query('select * from (select tbname, avg(f) from st partition by tbname) a partition by a.tbname order by a.tbname'); + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'ct1'); + tdSql.checkData(0, 1, 6.0); + tdSql.checkData(1, 0, 'ct2'); + tdSql.checkData(1, 1, 12.0); + tdSql.error('select tbname from (select * from st)') tdSql.error('select st.tbname from (select st.tbname from st)') tdSql.error('select `st.tbname` from (select st.tbname from st) order by tbname') From 4a77a49a3e3c38d6519390eb6d00df15496a9d9b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:25:48 +0800 Subject: [PATCH 038/177] prefix/list: list bucket with prefix --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +-- source/dnode/vnode/src/vnd/vnodeCos.c | 93 ++++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 93d0eff41a..790a6fe312 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl xml2 openssl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 24ad63db13..63a60fa59a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,7 +14,6 @@ set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -192,6 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..d4ea90f48a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PUBLIC ${S3_LIBRARY} - PUBLIC ${CURL_LIBRARY} - PUBLIC ${SSL_LIBRARY} - PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PRIVATE ${S3_LIBRARY} + PRIVATE ${CURL_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE xml2 ) add_definitions(-DUSE_S3) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 8984d33ad8..cf19a08813 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -63,9 +63,17 @@ static int should_retry() { return 0; } +static void s3PrintError(const char *func, S3Status status, char error_details[]) { + if (status < S3StatusErrorAccessDenied) { + vError("%s: %s", __func__, S3_get_status_name(status)); + } else { + vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + } +} + typedef struct { uint64_t content_length; - int status; + S3Status status; char *buf; char err_msg[4096]; } TS3SizeCBD; @@ -109,7 +117,86 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } -void s3DeleteObjectsByPrefix(const char *prefix) {} + +typedef struct list_bucket_callback_data { + int isTruncated; + char nextMarker[1024]; + int keyCount; + int allDetails; + S3Status status; + char err_msg[4096]; +} list_bucket_callback_data; + +static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, + const S3ListBucketContent *contents, int commonPrefixesCount, + const char **commonPrefixes, void *callbackData) { + list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData; + + data->isTruncated = isTruncated; + if ((!nextMarker || !nextMarker[0]) && contentsCount) { + nextMarker = contents[contentsCount - 1].key; + } + if (nextMarker) { + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); + } else { + data->nextMarker[0] = 0; + } + + if (contentsCount && !data->keyCount) { + // printListBucketHeader(data->allDetails); + } + + int i; + for (i = 0; i < contentsCount; ++i) { + const S3ListBucketContent *content = &(contents[i]); + printf("%-50s", content->key); + } + data->keyCount += contentsCount; + + for (i = 0; i < commonPrefixesCount; i++) { + // printf("\nCommon Prefix: %s\n", commonPrefixes[i]); + } + + return S3StatusOK; +} + +void s3DeleteObjectsByPrefix(const char *prefix) { + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &listBucketCallback}; + + const char *marker = 0, *delimiter = 0; + int maxkeys = 0, allDetails = 0; + list_bucket_callback_data data; + + if (marker) { + snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); + } else { + data.nextMarker[0] = 0; + } + data.keyCount = 0; + data.allDetails = allDetails; + + do { + data.isTruncated = 0; + do { + S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, + &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); + + if (data.status == S3StatusOK) { + if (!data.keyCount) { + // printListBucketHeader(allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + } +} void s3DeleteObjects(const char *object_name[], int nobject) { int status = 0; @@ -124,7 +211,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__func__, cbd.status, cbd.err_msg); } } } From 2479bb343e56f98f37ad749e6607d2449b8892a7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:54:51 +0800 Subject: [PATCH 039/177] prefix/delete: delete with prefix --- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/src/vnd/vnodeCos.c | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 63a60fa59a..b40a7101c3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -191,7 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index cf19a08813..726d78e167 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -123,6 +123,7 @@ typedef struct list_bucket_callback_data { char nextMarker[1024]; int keyCount; int allDetails; + SArray *objectArray; S3Status status; char err_msg[4096]; } list_bucket_callback_data; @@ -149,7 +150,9 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int int i; for (i = 0; i < contentsCount; ++i) { const S3ListBucketContent *content = &(contents[i]); - printf("%-50s", content->key); + // printf("%-50s", content->key); + char *object_key = strdup(content->key); + taosArrayPush(data->objectArray, &object_key); } data->keyCount += contentsCount; @@ -160,6 +163,11 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int return S3StatusOK; } +static void s3FreeObjectKey(void *pItem) { + char *key = (char *)pItem; + taosMemoryFree(key); +} + void s3DeleteObjectsByPrefix(const char *prefix) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; @@ -169,7 +177,11 @@ void s3DeleteObjectsByPrefix(const char *prefix) { const char *marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; - + data.objectArray = taosArrayInit(32, POINTER_BYTES); + if (!data.objectArray) { + vError("%s: %s", __func__, "out of memoty"); + return; + } if (marker) { snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); } else { @@ -192,10 +204,13 @@ void s3DeleteObjectsByPrefix(const char *prefix) { if (data.status == S3StatusOK) { if (!data.keyCount) { // printListBucketHeader(allDetails); + s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); } } else { s3PrintError(__func__, data.status, data.err_msg); } + + taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); } void s3DeleteObjects(const char *object_name[], int nobject) { From 7a23615b3fda7553c0ad17f6cee90f6b4d56920d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Oct 2023 17:53:48 +0800 Subject: [PATCH 040/177] fix:offset set to earliest clearly --- utils/test/c/replay_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c index c105670733..92f3fe5102 100644 --- a/utils/test/c/replay_test.c +++ b/utils/test/c/replay_test.c @@ -27,6 +27,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "enable.replay", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); From 93b99e4573e315f1a5bd5d0133738c44d03a10f9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 16 Oct 2023 18:21:19 +0800 Subject: [PATCH 041/177] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9a888c1225..b221d99666 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -172,9 +172,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 6 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 +e ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py From 0a84d7a8b3780b3a243ccdb556f88ea19fb72d28 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 Oct 2023 19:20:48 +0800 Subject: [PATCH 042/177] fix stream snap deadlock --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 09fffa1f74..c6255be7cb 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -198,8 +198,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { taosWLockLatch(&pTq->pStreamMeta->lock); tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode)); - - taosWLockLatch(&pTq->pStreamMeta->lock); if (rollback) { tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); } else { From f5ea3649c5a4a78266c7a19bc980d040c691ee0f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 Oct 2023 19:26:08 +0800 Subject: [PATCH 043/177] fix stream snap deadlock --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index c6255be7cb..a406b8df34 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -211,8 +211,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { goto _err; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); - if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { code = -1; taosMemoryFree(pWriter); From b133418d88d43918c67f7363b83306c02a030fdf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 Oct 2023 20:03:44 +0800 Subject: [PATCH 044/177] fix stream snap deadlock --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index a406b8df34..e122cf19d3 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -206,10 +206,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { code = tdbPostCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); if (code) goto _err; } - if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - code = -1; - goto _err; - } if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { code = -1; From 38c1f7d48d0b692e29a32c657f948b9901f8ead3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 16 Oct 2023 20:20:04 +0800 Subject: [PATCH 045/177] fix: add lock for trans --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index f4236964ca..e5f46c03cd 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - TdThreadMutex mutex; + int8_t lock; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1d8dd5e345..f8d6cba171 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -470,7 +470,6 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } - (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -543,10 +542,6 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; - } else { - #ifdef WINDOWS - taosThreadMutexInit(&pTrans->mutex, NULL); - #endif } return pTrans; } @@ -582,7 +577,6 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); - taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { @@ -1264,10 +1258,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - taosThreadMutexLock(&pTrans->mutex); + if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; if (pTrans->redoActionPos >= numOfActions) { - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } @@ -1339,7 +1333,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - taosThreadMutexUnlock(&pTrans->mutex); + atomic_store_8(&pTrans->lock, 0); return code; } From a30eedafecc8a320afb0fac4dbb25a115c532ecd Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:06:06 +0800 Subject: [PATCH 046/177] fix: trans mutex init --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e5f46c03cd..f4236964ca 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct { SArray* pRpcArray; SRWLatch lockRpcArray; int64_t mTraceId; - int8_t lock; + TdThreadMutex mutex; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f8d6cba171..76d6e21c64 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -428,6 +428,8 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), pTrans->startFunc); + taosThreadMutexInit(&pTrans->mutex, NULL); + if (pTrans->startFunc > 0) { TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); if (fp) { @@ -1258,10 +1260,10 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; - if (atomic_val_compare_exchange_8(&pTrans->lock, 0, 1) != 0) return code; + taosThreadMutexLock(&pTrans->mutex); if (pTrans->redoActionPos >= numOfActions) { - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } @@ -1333,7 +1335,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } } - atomic_store_8(&pTrans->lock, 0); + taosThreadMutexUnlock(&pTrans->mutex); return code; } From 29c6daaff5efd137cc9d16f5aaa5a5869a766e45 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Oct 2023 11:12:52 +0800 Subject: [PATCH 047/177] fix: trans mutex init --- source/dnode/mnode/impl/src/mndTrans.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 76d6e21c64..29a8ae1f29 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -472,6 +472,7 @@ void mndTransDropData(STrans *pTrans) { pTrans->param = NULL; pTrans->paramLen = 0; } + (void)taosThreadMutexDestroy(&pTrans->mutex); } static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { @@ -579,6 +580,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); + taosThreadMutexInit(&pTrans->mutex, NULL); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || pTrans->pRpcArray == NULL) { From 1459c82d75ef5d7cd43e525c7c55dcd0885e4513 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 11:23:21 +0800 Subject: [PATCH 048/177] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 4 ++-- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b221d99666..66ffc7f6dd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -171,13 +171,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 e ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 086fe6e4c5..c76188eee2 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -184,7 +184,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0] or expectrowcnt <= resultList[0]: + if expectrowcnt / 2 >= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 4ca897246ce2ba883c92f4c417ccc232976b56e3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 12 Oct 2023 20:32:04 +0800 Subject: [PATCH 049/177] enh: proceed sync log buffer on failure of appending too --- source/libs/sync/src/syncMain.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index edecfcb2bc..eca499cf28 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2860,11 +2860,12 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){ } int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { + int32_t code = -1; if (pEntry->dataLen < sizeof(SMsgHead)) { sError("vgId:%d, cannot append an invalid client request with no msg head. type:%s, dataLen:%d", ths->vgId, TMSG_INFO(pEntry->originalRpcType), pEntry->dataLen); syncEntryDestroy(pEntry); - return -1; + goto _out; } // append to log buffer @@ -2873,9 +2874,11 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { ASSERT(terrno != 0); (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno, false); syncEntryDestroy(pEntry); - return -1; + goto _out; } - + + code = 0; +_out:; // proceed match index, with replicating on needed SyncIndex matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, NULL, "Append"); @@ -2886,7 +2889,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { // multi replica if (ths->replicaNum > 1) { - return 0; + return code; } // single replica @@ -2894,10 +2897,10 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit until commitIndex:%" PRId64 "", ths->vgId, ths->commitIndex); - return -1; + code = -1; } - return 0; + return code; } bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { From 8fa8c40391fbea4dbb668202dabb9160a281d289 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 14:59:03 +0800 Subject: [PATCH 050/177] cos/put: simple object putting(< 5 gigabytes) --- source/dnode/vnode/src/vnd/vnodeCos.c | 250 +++++++++++++++++++++++++- 1 file changed, 249 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 726d78e167..2a36a898e0 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -116,7 +116,255 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } } -int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } +typedef struct growbuffer { + // The total number of bytes, and the start byte + int size; + // The start byte + int start; + // The blocks + char data[64 * 1024]; + struct growbuffer *prev, *next; +} growbuffer; + +// returns nonzero on success, zero on out of memory +static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) { + int origDataLen = dataLen; + while (dataLen) { + growbuffer *buf = *gb ? (*gb)->prev : 0; + if (!buf || (buf->size == sizeof(buf->data))) { + buf = (growbuffer *)malloc(sizeof(growbuffer)); + if (!buf) { + return 0; + } + buf->size = 0; + buf->start = 0; + if (*gb && (*gb)->prev) { + buf->prev = (*gb)->prev; + buf->next = *gb; + (*gb)->prev->next = buf; + (*gb)->prev = buf; + } else { + buf->prev = buf->next = buf; + *gb = buf; + } + } + + int toCopy = (sizeof(buf->data) - buf->size); + if (toCopy > dataLen) { + toCopy = dataLen; + } + + memcpy(&(buf->data[buf->size]), data, toCopy); + + buf->size += toCopy, data += toCopy, dataLen -= toCopy; + } + + return origDataLen; +} + +static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) { + *amtReturn = 0; + + growbuffer *buf = *gb; + + if (!buf) { + return; + } + + *amtReturn = (buf->size > amt) ? amt : buf->size; + + memcpy(buffer, &(buf->data[buf->start]), *amtReturn); + + buf->start += *amtReturn, buf->size -= *amtReturn; + + if (buf->size == 0) { + if (buf->next == buf) { + *gb = 0; + } else { + *gb = buf->next; + buf->prev->next = buf->next; + buf->next->prev = buf->prev; + } + free(buf); + buf = NULL; + } +} + +static void growbuffer_destroy(growbuffer *gb) { + growbuffer *start = gb; + + while (gb) { + growbuffer *next = gb->next; + free(gb); + gb = (next == start) ? 0 : next; + } +} + +typedef struct put_object_callback_data { + // FILE *infile; + TdFilePtr infileFD; + growbuffer *gb; + uint64_t contentLength, originalContentLength; + uint64_t totalContentLength, totalOriginalContentLength; + int noStatus; + S3Status status; + char err_msg[4096]; +} put_object_callback_data; + +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + + return ret; +} + +#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M + +typedef struct UploadManager { + // used for initial multipart + char *upload_id; + + // used for upload part object + char **etags; + int next_etags_pos; + + // used for commit Upload + growbuffer *gb; + int remaining; +} UploadManager; + +typedef struct MultipartPartData { + put_object_callback_data put_object_data; + int seq; + UploadManager *manager; +} MultipartPartData; + +S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + manager->upload_id = strdup(upload_id); + return S3StatusOK; +} + +S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { + responsePropertiesCallback(properties, callbackData); + + MultipartPartData *data = (MultipartPartData *)callbackData; + int seq = data->seq; + const char *etag = properties->eTag; + data->manager->etags[seq - 1] = strdup(etag); + data->manager->next_etags_pos = seq; + return S3StatusOK; +} + +static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { + UploadManager *manager = (UploadManager *)callbackData; + int ret = 0; + + if (manager->remaining) { + int toRead = ((manager->remaining > bufferSize) ? bufferSize : manager->remaining); + growbuffer_read(&(manager->gb), toRead, &ret, buffer); + } + manager->remaining -= ret; + return ret; +} + +static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { + // + + return 0; +} + +int32_t s3PutObjectFromFile2(const char *file, const char *object) { + int32_t code = 0; + const char *key = object; + const char *uploadId = 0; + const char *filename = 0; + uint64_t contentLength = 0; + const char *cacheControl = 0, *contentType = 0, *md5 = 0; + const char *contentDispositionFilename = 0, *contentEncoding = 0; + int64_t expires = -1; + S3CannedAcl cannedAcl = S3CannedAclPrivate; + int metaPropertiesCount = 0; + S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; + char useServerSideEncryption = 0; + int noStatus = 0; + put_object_callback_data data; + + // data.infile = 0; + data.infileFD = NULL; + data.gb = 0; + data.noStatus = noStatus; + + if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { + vError("ERROR: %s Failed to stat file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { + vError("ERROR: %s Failed to open file %s: ", __func__, file); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = + contentLength; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3PutProperties putProperties = {contentType, md5, + cacheControl, contentDispositionFilename, + contentEncoding, expires, + cannedAcl, metaPropertiesCount, + metaProperties, useServerSideEncryption}; + + if (contentLength <= MULTIPART_CHUNK_SIZE) { + S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + do { + S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + + if (data.status != S3StatusOK) { + s3PrintError(__func__, data.status, data.err_msg); + } else if (data.contentLength) { + vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, + (unsigned long long)data.contentLength); + } + } else { + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + } + return 0; +} typedef struct list_bucket_callback_data { int isTruncated; From d8b0d8085be4cc51903d489c915a2a8a5331b9d4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 15:16:20 +0800 Subject: [PATCH 051/177] cmake/ssl: make curl depends openssl --- cmake/curl_CMakeLists.txt.in | 1 + cmake/libs3_CMakeLists.txt.in | 4 ++-- contrib/CMakeLists.txt | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 24a1210422..fed98c5f0e 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -7,6 +7,7 @@ ExternalProject_Add(curl #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 #UPDATE_COMMAND "" diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 790a6fe312..b2d2d19059 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -12,8 +12,8 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 - #GIT_TAG v5.0.16 - DEPENDS curl xml2 openssl + #GIT_TAG v5.0.16 + DEPENDS curl xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b40a7101c3..07201f8713 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -277,7 +277,6 @@ if(${BUILD_TEST}) ) endif(${TD_DARWIN}) - endif(${BUILD_TEST}) # cJson From ddfdbb3207415134c8b5c684864debe6da716848 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 15:28:36 +0800 Subject: [PATCH 052/177] xml2: add sub dir if building libs3 --- contrib/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 07201f8713..de3d9cfd37 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -24,7 +24,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) + #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -294,7 +294,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -add_subdirectory(xml2 EXCLUDE_FROM_ALL) +if(${BUILD_WITH_S3}) + add_subdirectory(xml2 EXCLUDE_FROM_ALL) +endif(${BUILD_WITH_S3}) # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) From a16c9be49f41f8c4233171f4dda235e41e35ca6d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 15:34:16 +0800 Subject: [PATCH 053/177] feat:[TD-19291]Schemaless table name can be composed based on rule --- .../03-insert-data/30-influxdb-line.mdx | 5 +- .../03-insert-data/40-opentsdb-telnet.mdx | 5 +- .../03-insert-data/50-opentsdb-json.mdx | 5 +- docs/en/14-reference/12-config/index.md | 9 +++ .../13-schemaless/13-schemaless.md | 2 + ...-influxdb-line.mdx => 30-influxdb-line.md} | 5 +- .../03-insert-data/40-opentsdb-telnet.mdx | 5 +- .../03-insert-data/50-opentsdb-json.mdx | 5 +- docs/zh/14-reference/12-config/index.md | 11 +++- .../13-schemaless/13-schemaless.md | 7 ++- examples/rust/src/lib.rs | 11 ++++ include/common/tglobal.h | 1 + source/client/src/clientSml.c | 52 ++++++++++++---- source/common/src/tglobal.c | 9 ++- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/sml-TD19291.py | 60 +++++++++++++++++++ tests/system-test/2-query/sml.py | 2 +- utils/test/c/sml_test.c | 4 +- 18 files changed, 172 insertions(+), 27 deletions(-) rename docs/zh/07-develop/03-insert-data/{30-influxdb-line.mdx => 30-influxdb-line.md} (76%) create mode 100644 examples/rust/src/lib.rs create mode 100644 tests/system-test/2-query/sml-TD19291.py diff --git a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx index c85363c9db..bd430d5973 100644 --- a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx @@ -38,7 +38,10 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to NCHAR type automatically - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h) -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored +- The rule of table name + - The child table name is created automatically in a rule to guarantee its uniqueness. + - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4` + - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, smlDataFormat is discarded since 3.0.3.0) ::: diff --git a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx index 1147ce01b0..ed2659042f 100644 --- a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx @@ -33,7 +33,10 @@ For example: meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The rule of table name + - The child table name is created automatically in a rule to guarantee its uniqueness. + - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4` + - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. diff --git a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx index 3dd6c6b756..a40b5f264d 100644 --- a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -48,7 +48,10 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http :::note - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The rule of table name + - The child table name is created automatically in a rule to guarantee its uniqueness. + - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4` + - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored ::: diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 011db81b92..ac2364a65d 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -652,6 +652,15 @@ The charset that takes effect is UTF-8. | Type | String | | Default Value | None | +### smlAutoChildTableNameDelimiter + +| Attribute | Description | +| ------------- | ------------------------------------------ | +| Applicable | Client only | +| Meaning | Delimiter between tags as table name| +| Type | String | +| Default Value | None | + ### smlTagName | Attribute | Description | diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index d4b6606ac5..11d3c0d0ab 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -93,6 +93,8 @@ Note that tag_key1, tag_key2 are not the original order of the tags entered by t The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has. ::: +If you do not want to use an automatically generated table name, there are two ways to specify sub table names, the first one has a higher priority. +You can configure smlAutoChildTableNameDelimiter in taos.cfg, for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. diff --git a/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/30-influxdb-line.md similarity index 76% rename from docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx rename to docs/zh/07-develop/03-insert-data/30-influxdb-line.md index eb17386fd0..ed84f05fef 100644 --- a/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/30-influxdb-line.md @@ -38,7 +38,10 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理 - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度 - 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false，从3.0.3.0开始，该配置废弃） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) -- 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 子表名生成规则 + - 默认产生的子表名是根据规则生成的唯一 ID 值。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: diff --git a/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx index 97cb0d4a11..195f3a31f4 100644 --- a/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx @@ -31,8 +31,11 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB ```txt meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` +- 子表名生成规则 + - 默认产生的子表名是根据规则生成的唯一 ID 值。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) -- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 参考 [OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。 ## 示例代码 diff --git a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx index a63579bb2c..ce8d5b0caa 100644 --- a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,10 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 :::note - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 -- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 +- 子表名生成规则 + - 默认产生的子表名是根据规则生成的唯一 ID 值。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index c7323da271..27a9618107 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -648,7 +648,16 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅客户端适用 | | 含义 | schemaless 自定义的子表名的 key | | 类型 | 字符串 | -| 缺省值 | 无 | +| 缺省值 | 无 + +### smlAutoChildTableNameDelimiter + +| 属性 | 说明 | +| -------- | ------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | schemaless tag之间的连接符，连起来作为子表名 | +| 类型 | 字符串 | +| 缺省值 | 无 | ### smlTagName diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 969bc8c2ae..084e31d810 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -94,8 +94,11 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 :::tip 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 -:::tip -为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +:::tip +如果不想用自动生成的表名，有两种指定子表名的方式，第一种优先级更高： +通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定。 +举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 +通过在taos.cfg里配置 smlChildTableName 参数来指定。 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。 diff --git a/examples/rust/src/lib.rs b/examples/rust/src/lib.rs new file mode 100644 index 0000000000..4f8a5f1df8 --- /dev/null +++ b/examples/rust/src/lib.rs @@ -0,0 +1,11 @@ +#![allow(unused)] +#![allow(non_camel_case_types)] + +#[path = "bindings.rs"] +pub mod bindings; +#[path = "subscriber.rs"] +pub mod subscriber; +#[path = "tdengine.rs"] +pub mod tdengine; +#[path = "utils.rs"] +pub mod utils; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4644c38ec4..c4037ed2ea 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -179,6 +179,7 @@ extern char tsUdfdLdLibPath[]; // schemaless extern char tsSmlChildTableName[]; +extern char tsSmlAutoChildTableNameDelimiter[]; extern char tsSmlTagName[]; extern bool tsSmlDot2Underline; extern char tsSmlTsDefaultName[]; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 91c21fe344..17645fbec5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -193,20 +193,46 @@ cleanup: } static int32_t smlParseTableName(SArray *tags, char *childTableName) { - size_t childTableNameLen = strlen(tsSmlChildTableName); - if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; - - for (int i = 0; i < taosArrayGetSize(tags); i++) { - SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - // handle child table name - if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); - if(tsSmlDot2Underline){ - smlStrReplace(childTableName, strlen(childTableName)); + bool autoChildName = false; + size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); + if(delimiter > 0){ + size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + totalNameLen += tag->length; + } + if(totalNameLen < TSDB_TABLE_NAME_LEN){ + autoChildName = true; + } + } + if(autoChildName){ + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + strncat(childTableName, tag->value, tag->length); + if(i != taosArrayGetSize(tags) - 1){ + strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + } + } + if(tsSmlDot2Underline){ + smlStrReplace(childTableName, strlen(childTableName)); + } + }else{ + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + if(tsSmlDot2Underline){ + smlStrReplace(childTableName, strlen(childTableName)); + } + taosArrayRemove(tags, i); + break; } - taosArrayRemove(tags, i); - break; } } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3d7b38161a..ed1776dc45 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -118,7 +118,8 @@ bool tsSmlDot2Underline = true; char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts"; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. - // If set to empty system will generate table name using MD5 hash. +char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; + // If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) // bool tsSmlDataFormat = false; // int32_t tsSmlBatchSize = 10000; @@ -439,7 +440,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlChildTableName", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1; @@ -931,6 +933,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } + tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN); @@ -1396,6 +1399,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); } else if (strcasecmp("smlChildTableName", name) == 0) { tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) { + tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN); } else if (strcasecmp("smlTagName", name) == 0) { tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); // } else if (strcasecmp("smlDataFormat", name) == 0) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e83586ca09..9e596fc7af 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -399,6 +399,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TD19291.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R diff --git a/tests/system-test/2-query/sml-TD19291.py b/tests/system-test/2-query/sml-TD19291.py new file mode 100644 index 0000000000..7d13cecb0f --- /dev/null +++ b/tests/system-test/2-query/sml-TD19291.py @@ -0,0 +1,60 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'smlAutoChildTableNameDelimiter': '-', 'fqdn': 'localhost', 'smlDot2Underline': 1}, 'fqdn': 'localhost'} + print("===================: ", updatecfgDict) + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def check(self): + conn = taos.connect() + dbname = "td19291" + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s precision 'us'" % dbname) + conn.select_db(dbname) + + lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t1=3i64,t2=4f64,dataModelName=ttt c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t1=3i64,t2=4f.64 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t1=ioiooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t2=q,t1=iooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t2=a,t1=ooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + ] + conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) + print("inserted") + + tdSql.query("select table_name from information_schema.ins_tables where type = 'CHILD_TABLE' order by table_name") + tdSql.checkRows(6) + tdSql.checkData(0, 0, "3i64-4f64-\"t3\"") + tdSql.checkData(1, 0, "3i64-4f64-ttt") + tdSql.checkData(2, 0, "3i64-4f_64") + tdSql.checkData(3, 0, "a-ooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4") + tdSql.checkData(4, 0, "t_418c134a0f00c7f536886e132d5fbfff") + tdSql.checkData(5, 0, "t_cb0dbf4ee9c9052815c17fc6483b0139") + # tdSql.query(f"select * from td24559.stb order by _ts") + # tdSql.checkRows(4) + # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)") + # tdSql.checkData(3, 2, "GEOMETRYCOLLECTION (MULTIPOINT ((0.000000 0.000000), (1.000000 1.000000)), POINT (3.000000 4.000000), LINESTRING (2.000000 3.000000, 3.000000 4.000000))") + return + + def run(self): + tdSql.prepare() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 53ac85bd8e..0369f45723 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -9,7 +9,7 @@ from util.dnodes import * from util.common import * class TDTestCase: - updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'} + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'smlAutoChildTableNameDelimiter': '', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'} print("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9153706d23..64da4f83e3 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1678,8 +1678,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_td18789_Test(); ASSERT(!ret); - ret = sml_td24070_Test(); - ASSERT(!ret); +// ret = sml_td24070_Test(); +// ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); From 486227d34157452e166c748a4bdc9e46f89cb02f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 15:35:49 +0800 Subject: [PATCH 054/177] fix:rollback --- examples/rust/src/lib.rs | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 examples/rust/src/lib.rs diff --git a/examples/rust/src/lib.rs b/examples/rust/src/lib.rs deleted file mode 100644 index 4f8a5f1df8..0000000000 --- a/examples/rust/src/lib.rs +++ /dev/null @@ -1,11 +0,0 @@ -#![allow(unused)] -#![allow(non_camel_case_types)] - -#[path = "bindings.rs"] -pub mod bindings; -#[path = "subscriber.rs"] -pub mod subscriber; -#[path = "tdengine.rs"] -pub mod tdengine; -#[path = "utils.rs"] -pub mod utils; From d8c48dd3465ca5f02af4c4723ce1261a8d1c7990 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 15:36:59 +0800 Subject: [PATCH 055/177] fix:rollback --- .../03-insert-data/{30-influxdb-line.md => 30-influxdb-line.mdx} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/zh/07-develop/03-insert-data/{30-influxdb-line.md => 30-influxdb-line.mdx} (100%) diff --git a/docs/zh/07-develop/03-insert-data/30-influxdb-line.md b/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx similarity index 100% rename from docs/zh/07-develop/03-insert-data/30-influxdb-line.md rename to docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx From dd21d8e3ade60225d74d2cf37ea49fe4027bf9ba Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 17:24:50 +0800 Subject: [PATCH 056/177] cos/multipart: first round multipart uploading --- source/dnode/vnode/src/vnd/vnodeCos.c | 256 ++++++++++++++++++++++++-- 1 file changed, 244 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 2a36a898e0..b52d4809b2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -72,10 +72,10 @@ static void s3PrintError(const char *func, S3Status status, char error_details[] } typedef struct { - uint64_t content_length; + char err_msg[128]; S3Status status; + uint64_t content_length; char *buf; - char err_msg[4096]; } TS3SizeCBD; static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { @@ -201,14 +201,14 @@ static void growbuffer_destroy(growbuffer *gb) { } typedef struct put_object_callback_data { + char err_msg[128]; + S3Status status; // FILE *infile; TdFilePtr infileFD; growbuffer *gb; uint64_t contentLength, originalContentLength; uint64_t totalContentLength, totalOriginalContentLength; int noStatus; - S3Status status; - char err_msg[4096]; } put_object_callback_data; static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { @@ -241,6 +241,8 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { + char err_msg[128]; + S3Status status; // used for initial multipart char *upload_id; @@ -253,7 +255,26 @@ typedef struct UploadManager { int remaining; } UploadManager; +typedef struct list_parts_callback_data { + char err_msg[128]; + S3Status status; + int isTruncated; + char nextPartNumberMarker[24]; + char initiatorId[256]; + char initiatorDisplayName[256]; + char ownerId[256]; + char ownerDisplayName[256]; + char storageClass[256]; + int partsCount; + int handlePartsStart; + int allDetails; + int noPrint; + UploadManager *manager; +} list_parts_callback_data; + typedef struct MultipartPartData { + char err_msg[128]; + S3Status status; put_object_callback_data put_object_data; int seq; UploadManager *manager; @@ -288,16 +309,120 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD return ret; } +static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, + const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, + const char *storageClass, int partsCount, int handlePartsStart, + const S3ListPart *parts, void *callbackData) { + list_parts_callback_data *data = (list_parts_callback_data *)callbackData; + + data->isTruncated = isTruncated; + data->handlePartsStart = handlePartsStart; + UploadManager *manager = data->manager; + + if (nextPartNumberMarker) { + snprintf(data->nextPartNumberMarker, sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker); + } else { + data->nextPartNumberMarker[0] = 0; + } + + if (initiatorId) { + snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId); + } else { + data->initiatorId[0] = 0; + } + + if (initiatorDisplayName) { + snprintf(data->initiatorDisplayName, sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName); + } else { + data->initiatorDisplayName[0] = 0; + } + + if (ownerId) { + snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId); + } else { + data->ownerId[0] = 0; + } + + if (ownerDisplayName) { + snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName); + } else { + data->ownerDisplayName[0] = 0; + } + + if (storageClass) { + snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass); + } else { + data->storageClass[0] = 0; + } + + if (partsCount && !data->partsCount && !data->noPrint) { + // printListPartsHeader(); + } + + int i; + for (i = 0; i < partsCount; i++) { + const S3ListPart *part = &(parts[i]); + char timebuf[256]; + if (data->noPrint) { + manager->etags[handlePartsStart + i] = strdup(part->eTag); + manager->next_etags_pos++; + manager->remaining = manager->remaining - part->size; + } else { + time_t t = (time_t)part->lastModified; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); + printf("%-30s", timebuf); + printf("%-15llu", (unsigned long long)part->partNumber); + printf("%-45s", part->eTag); + printf("%-15llu\n", (unsigned long long)part->size); + } + } + + data->partsCount += partsCount; + + return S3StatusOK; +} + static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { - // + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback}; + + list_parts_callback_data data; + + memset(&data, 0, sizeof(list_parts_callback_data)); + + data.partsCount = 0; + data.allDetails = 0; + data.manager = manager; + data.noPrint = 1; + do { + data.isTruncated = 0; + do { + S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, manager->upload_id, 0, 0, 0, timeoutMsG, + &listPartsHandler, &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated); + + if (data.status == S3StatusOK) { + if (!data.partsCount) { + // printListMultipartHeader(data.allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + return -1; + } return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object) { - int32_t code = 0; - const char *key = object; - const char *uploadId = 0; + int32_t code = 0; + const char *key = object; + // const char *uploadId = 0; const char *filename = 0; uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; @@ -360,20 +485,127 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { (unsigned long long)data.contentLength); } } else { - uint64_t totalContentLength = contentLength; - uint64_t todoContentLength = contentLength; + uint64_t totalContentLength = contentLength; + uint64_t todoContentLength = contentLength; + UploadManager manager; + manager.upload_id = 0; + manager.gb = 0; + + // div round up + int seq; + int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); + + MultipartPartData partData; + memset(&partData, 0, sizeof(MultipartPartData)); + int partContentLength = 0; + + S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &initial_multipart_callback}; + + S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, + &putObjectDataCallback}; + + S3MultipartCommitHandler commit_handler = { + {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + + manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); + manager.next_etags_pos = 0; + /* + if (uploadId) { + manager.upload_id = strdup(uploadId); + manager.remaining = contentLength; + if (!try_get_parts_info(tsS3BucketName, key, &manager)) { + fseek(data.infile, -(manager.remaining), 2); + taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END); + contentLength = manager.remaining; + goto upload; + } else { + goto clean; + } + } + */ + do { + S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + + if (manager.upload_id == 0 || manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + goto clean; + } + + upload: + todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; + for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { + partData.manager = &manager; + partData.seq = seq; + if (partData.put_object_data.gb == NULL) { + partData.put_object_data = data; + } + partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); + // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); + partData.put_object_data.contentLength = partContentLength; + partData.put_object_data.originalContentLength = partContentLength; + partData.put_object_data.totalContentLength = todoContentLength; + partData.put_object_data.totalOriginalContentLength = totalContentLength; + putProperties.md5 = 0; + do { + S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, + partContentLength, 0, timeoutMsG, &partData); + } while (S3_status_is_retryable(partData.status) && should_retry()); + if (partData.status != S3StatusOK) { + s3PrintError(__func__, partData.status, partData.err_msg); + goto clean; + } + contentLength -= MULTIPART_CHUNK_SIZE; + todoContentLength -= MULTIPART_CHUNK_SIZE; + } + + int i; + int size = 0; + size += growbuffer_append(&(manager.gb), "", strlen("")); + char buf[256]; + int n; + for (i = 0; i < totalSeq; i++) { + n = snprintf(buf, sizeof(buf), + "%d" + "%s", + i + 1, manager.etags[i]); + size += growbuffer_append(&(manager.gb), buf, n); + } + size += growbuffer_append(&(manager.gb), "", strlen("")); + manager.remaining = size; + + do { + S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0, + timeoutMsG, &manager); + } while (S3_status_is_retryable(manager.status) && should_retry()); + if (manager.status != S3StatusOK) { + s3PrintError(__func__, manager.status, manager.err_msg); + goto clean; + } + + clean: + if (manager.upload_id) { + taosMemoryFree(manager.upload_id); + } + for (i = 0; i < manager.next_etags_pos; i++) { + taosMemoryFree(manager.etags[i]); + } + growbuffer_destroy(manager.gb); + taosMemoryFree(manager.etags); } + return 0; } typedef struct list_bucket_callback_data { + char err_msg[128]; + S3Status status; int isTruncated; char nextMarker[1024]; int keyCount; int allDetails; SArray *objectArray; - S3Status status; - char err_msg[4096]; } list_bucket_callback_data; static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, From 4bc3c22e0c92e8831d1cf6ee55bdb4be4ef6385b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 17:46:11 +0800 Subject: [PATCH 057/177] cmake/ssl: install install_sw only --- cmake/ssl_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 7821848618..65a02d7fd6 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -9,7 +9,7 @@ ExternalProject_Add(openssl #BUILD_ALWAYS 1 #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared - BUILD_COMMAND make -j - INSTALL_COMMAND make install + BUILD_COMMAND "" + INSTALL_COMMAND make install_sw -j TEST_COMMAND "" ) From 11b65e277925323b65efe49095a8c2cff3e72278 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 17 Oct 2023 18:11:13 +0800 Subject: [PATCH 058/177] vnode/cos: make dependent lib public --- cmake/curl_CMakeLists.txt.in | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index fed98c5f0e..3f7b9ee1fe 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -12,7 +12,7 @@ ExternalProject_Add(curl BUILD_ALWAYS 1 #UPDATE_COMMAND "" #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #CONFIGURE_COMMAND ./configure --without-ssl BUILD_COMMAND make INSTALL_COMMAND make install diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d4ea90f48a..e554d94e4d 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PRIVATE ${S3_LIBRARY} - PRIVATE ${CURL_LIBRARY} - PRIVATE ${SSL_LIBRARY} - PRIVATE ${CRYPTO_LIBRARY} - PRIVATE xml2 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC xml2 ) add_definitions(-DUSE_S3) From 6d9732e87a850b6aee3be6e6e77a16240861cd68 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 18:26:04 +0800 Subject: [PATCH 059/177] test: set taosd log leavel 135 --- tests/pytest/util/dnodes.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c4fc1ce654..d0ed1d874d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -36,12 +36,12 @@ class TDSimClient: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "rpcDebugFlag": "143", + "rpcDebugFlag": "135", "tmrDebugFlag": "131", - "cDebugFlag": "143", - "uDebugFlag": "143", - "jniDebugFlag": "143", - "qDebugFlag": "143", + "cDebugFlag": "135", + "uDebugFlag": "135", + "jniDebugFlag": "135", + "qDebugFlag": "135", "supportVnodes": "1024", "enableQueryHb": "1", "telemetryReporting": "0", @@ -130,18 +130,18 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "mDebugFlag": "143", - "dDebugFlag": "143", - "vDebugFlag": "143", - "tqDebugFlag": "143", - "cDebugFlag": "143", - "jniDebugFlag": "143", - "qDebugFlag": "143", - "rpcDebugFlag": "143", + "mDebugFlag": "135", + "dDebugFlag": "135", + "vDebugFlag": "135", + "tqDebugFlag": "135", + "cDebugFlag": "135", + "jniDebugFlag": "135", + "qDebugFlag": "135", + "rpcDebugFlag": "135", "tmrDebugFlag": "131", - "uDebugFlag": "143", - "sDebugFlag": "143", - "wDebugFlag": "143", + "uDebugFlag": "135", + "sDebugFlag": "135", + "wDebugFlag": "135", "numOfLogLines": "100000000", "statusInterval": "1", "enableQueryHb": "1", From d6579d0498bbfd4728c29196dd42c303e897f48c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 17 Oct 2023 18:30:47 +0800 Subject: [PATCH 060/177] test: support replica3 in splited/transform tmqvnode --- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 18f9fafe9a..006498d514 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -256,7 +256,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") def tmqCaseStableSelect(self): - tdLog.printNoPrefix("======== test case subscrib column start : ") + tdLog.printNoPrefix("======== test case 3 subscrib column start : ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', @@ -315,7 +315,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - time.sleep(5) + time.sleep(1) #restart dnode & remove wal # self.restartAndRemoveWal() @@ -332,10 +332,10 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - tdLog.printNoPrefix("======== test case subscrib column end ...... ") + tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") def tmqCaseDbname(self): - tdLog.printNoPrefix("======== test case subscrib Dbname start: ") + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', @@ -394,7 +394,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) tdLog.info("wait the consume result") - time.sleep(5) + time.sleep(1) #restart dnode & remove wal # self.restartAndRemoveWal() @@ -411,7 +411,7 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - tdLog.printNoPrefix("======== test case subscrib Dbname end ...... ") + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") def run(self): self.prepareTestEnv() From 5839e71f1fd7033e875aa9bf0661c0e790a690a1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 18:52:25 +0800 Subject: [PATCH 061/177] fix:test case error --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9e596fc7af..0d36de5553 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -399,7 +399,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TD19291.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml-TD19291.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R From cdc8b757ad6a58b9cfd1d9d509c8f2de9019abc3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 Oct 2023 10:11:43 +0800 Subject: [PATCH 062/177] fix: remove subquery user alias name distinct check --- source/libs/parser/src/parTranslater.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42ab2f7a2f..85d154b94c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3318,9 +3318,6 @@ static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjection } static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (pSelect->isSubquery) { - return checkProjectAlias(pCxt, pSelect->pProjectionList, NULL); - } return rewriteProjectAlias(pSelect->pProjectionList); } From a9dfb400691ee59d8e4d35fe3c97e649a5480250 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 10:51:47 +0800 Subject: [PATCH 063/177] contrib/cmake: remove unused steps --- contrib/CMakeLists.txt | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index de3d9cfd37..30853660ed 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -8,30 +8,9 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) - if(${BUILD_WITH_S3}) - -set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") -configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) - +if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) -configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") -execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - -set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2") -configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - - #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2}) - -configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") -execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - else() set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") @@ -185,12 +164,6 @@ endif(${BUILD_WITH_SQLITE}) # s3 if(${BUILD_WITH_S3}) - #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) From ed544186b3f6e473b0382821d57b386570a7047f Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 18 Oct 2023 10:53:31 +0800 Subject: [PATCH 064/177] test: add 500ms sleep after create-stream for branch-3.0 --- tests/pytest/util/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index ea042829d6..9c45c09715 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -140,6 +140,7 @@ class TDCom: self.range_count = 5 self.default_interval = 5 self.stream_timeout = 12 + self.create_stream_sleep = 0.5 self.record_history_ts = str() self.precision = "ms" self.date_time = self.genTs(precision=self.precision)[0] @@ -881,6 +882,7 @@ class TDCom: stream_options += f" ignore update 0" if not use_except: tdSql.execute(f'create stream if not exists {stream_name} trigger at_once {stream_options} {fill_history} into {des_table} {subtable} as {source_sql} {fill};') + time.sleep(self.create_stream_sleep) return None else: return f'create stream if not exists {stream_name} {stream_options} {fill_history} into {des_table} {subtable} as {source_sql} {fill};' @@ -906,6 +908,7 @@ class TDCom: stream_options += f" ignore update 0" if not use_except: tdSql.execute(f'create stream if not exists {stream_name} {stream_options} {fill_history} into {des_table}{stb_field_name} {tags} {subtable} as {source_sql} {fill};') + time.sleep(self.create_stream_sleep) return None else: return f'create stream if not exists {stream_name} {stream_options} {fill_history} into {des_table}{stb_field_name} {tags} {subtable} as {source_sql} {fill};' @@ -1566,8 +1569,8 @@ class TDCom: res1 = tdSql.queryResult tdSql.query(sql2) res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + tdSql.sql = sql1 new_list = list() - if tag_value_list: res1 = self.float_handle(res1) res2 = self.float_handle(res2) @@ -1602,6 +1605,7 @@ class TDCom: tdSql.query(sql2) # res2 = tdSql.queryResult res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + tdSql.sql = sql1 if tag_value_list: res1 = self.float_handle(res1) @@ -1643,6 +1647,7 @@ class TDCom: tdSql.query(sql2) # res2 = tdSql.queryResult res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + tdSql.sql = sql1 if tag_value_list: res1 = self.float_handle(res1) From 98d782cfa5d63479c8253c16388f7dde0beb3f41 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 Oct 2023 11:04:34 +0800 Subject: [PATCH 065/177] fix: if not subquery, rewrite project alias --- source/libs/parser/src/parTranslater.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 85d154b94c..05c5b427fe 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3318,7 +3318,9 @@ static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjection } static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { - return rewriteProjectAlias(pSelect->pProjectionList); + if (!pSelect->isSubquery) { + return rewriteProjectAlias(pSelect->pProjectionList); + } } static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { From 9c1697bfd11cb701e2bd82b97a78e1c5b6cfd879 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 Oct 2023 13:19:44 +0800 Subject: [PATCH 066/177] fix: fix minior error --- source/libs/parser/src/parTranslater.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 05c5b427fe..4d212a1c3d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3320,6 +3320,8 @@ static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjection static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { if (!pSelect->isSubquery) { return rewriteProjectAlias(pSelect->pProjectionList); + } else { + return TSDB_CODE_SUCCESS; } } From f26ec41b8e11804ba87524e5accc8cc5cd459af1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 15:30:02 +0800 Subject: [PATCH 067/177] vnode/cos: fix get callback buffer --- source/common/CMakeLists.txt | 3 +++ source/common/src/tglobal.c | 2 +- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/vnd/vnodeCos.c | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index b010467f20..6bdcdf7d1d 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -22,6 +22,9 @@ IF (TD_STORAGE) add_definitions(-DUSE_COS) ENDIF(${BUILD_WITH_COS}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ENDIF(${BUILD_WITH_S3}) ENDIF(${TD_LINUX}) ENDIF () diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8ea5528fbb..f3af6e0ab0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -329,7 +329,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } } if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) { -#ifdef USE_COS +#if defined(USE_COS) || defined(USE_S3) tsS3Enabled = true; #endif } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..7e2693e8af 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -164,7 +164,6 @@ target_link_libraries( if(${TD_LINUX}) if(${BUILD_WITH_S3}) - MESSAGE("build with s3: ${BUILD_WITH_S3}") target_include_directories( vnode diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index b52d4809b2..c50f133987 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -721,7 +721,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void * char *buf = taosMemoryCalloc(1, bufferSize); if (buf) { memcpy(buf, buffer, bufferSize); - + cbd->buf = buf; cbd->status = S3StatusOK; return S3StatusOK; } else { From 02658a4b3ca9febde23ec4daeeec5d5c4d5f62ab Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 Oct 2023 15:30:18 +0800 Subject: [PATCH 068/177] fix: add test case --- tests/script/tsim/query/unionall_as_table.sim | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim index f8145d4e97..11fdf17978 100644 --- a/tests/script/tsim/query/unionall_as_table.sim +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -1,5 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c keepColumnName -v 1 system sh/exec.sh -n dnode1 -s start sql connect @@ -50,4 +51,14 @@ print $data00 if $data00 != 2 then return -1 endi +sql select count(*) from (select f, f from ctcount) +print $data00 +if $data00 != 2 then + return -1 +endi +sql select count(*) from (select last(ts), first(ts) from ctcount); +print $data00 +if $data00 != 1 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From e7d9de06cb0f73f9fb5148f30694f675dfcfee03 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 18 Oct 2023 15:59:04 +0800 Subject: [PATCH 069/177] test: support replica3 in splited/transform tmqvnode --- .../system-test/7-tmq/tmqVnodeTransform-db.py | 298 ++++++++++++++++++ .../7-tmq/tmqVnodeTransform-stb.py | 165 +--------- 2 files changed, 300 insertions(+), 163 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-db.py diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py new file mode 100644 index 0000000000..005bca70d6 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -0,0 +1,298 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 30, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + + def tmqCaseStableSelect(self): + tdLog.printNoPrefix("======== test case 3 subscrib column start : ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic3'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") + + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + #restart dnode & remove wal + # self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCaseStableSelect() + self.prepareTestEnv() + self.tmqCaseDbname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 006498d514..2dd756b788 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -49,7 +49,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -140,7 +140,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 30, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -255,173 +255,12 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") - def tmqCaseStableSelect(self): - tdLog.printNoPrefix("======== test case 3 subscrib column start : ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stbn', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic3'] - tmqCom.initConsumerTable() - - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s where c2 > 0 "%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - time.sleep(1) - #restart dnode & remove wal - # self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups() - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - time.sleep(5) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 3 subscrib column end ...... ") - - def tmqCaseDbname(self): - tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stbn', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic4'] - tmqCom.initConsumerTable() - - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from database ") - queryString = "database %s "%(paraDict['dbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - time.sleep(1) - #restart dnode & remove wal - # self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups() - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - time.sleep(5) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") def run(self): self.prepareTestEnv() self.tmqCaseStable() self.prepareTestEnv() self.tmqCaseNtable() - self.prepareTestEnv() - self.tmqCaseStableSelect() - self.prepareTestEnv() - self.tmqCaseDbname() def stop(self): tdSql.close() From e62f3fa86a76a025ea5d8c53d8f42ad26ee8e86f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Oct 2023 16:03:55 +0800 Subject: [PATCH 070/177] opt status send --- include/common/tmsgcb.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 3 ++ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 5 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 53 +++++++++++++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 311bffb7da..7ab69759e3 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -52,6 +52,7 @@ typedef struct { void* data; void* mgmt; void* clientRpc; + void* statusClientRpc; void* serverRpc; PutToQueueFp putToQueueFp; GetQueueSizeFp qsizeFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index c7af552da4..3453731c03 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -160,7 +160,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecv(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp); + rpcSendRecv(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[256]; diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 3cf7a360f9..f11378bb71 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -48,6 +48,7 @@ typedef struct { typedef struct { void *serverRpc; void *clientRpc; + void *statusClientRpc; SDnodeHandle msgHandles[TDMT_MAX]; } SDnodeTrans; @@ -115,7 +116,9 @@ int32_t dmRunDnode(SDnode *pDnode); int32_t dmInitServer(SDnode *pDnode); void dmCleanupServer(SDnode *pDnode); int32_t dmInitClient(SDnode *pDnode); +int32_t dmInitStatusClient(SDnode *pDnode); void dmCleanupClient(SDnode *pDnode); +void dmCleanupStatusClient(SDnode *pDnode); SMsgCb dmGetMsgcb(SDnode *pDnode); int32_t dmInitMsgHandle(SDnode *pDnode); int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 15697dc448..5164d60ba6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -20,8 +20,8 @@ #include "qworker.h" #include "tstream.h" #ifdef TD_TSZ -#include "tglobal.h" #include "tcompression.h" +#include "tglobal.h" #endif int32_t dmInitDnode(SDnode *pDnode) { @@ -66,7 +66,7 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } - if(dmInitModule(pDnode) != 0) { + if (dmInitModule(pDnode) != 0) { goto _OVER; } @@ -91,6 +91,7 @@ void dmCleanupDnode(SDnode *pDnode) { if (pDnode == NULL) return; dmCleanupClient(pDnode); + dmCleanupStatusClient(pDnode); dmCleanupServer(pDnode); dmClearVars(pDnode); rpcCleanup(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index dc48ff71f8..b273706feb 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -358,6 +358,50 @@ int32_t dmInitClient(SDnode *pDnode) { dDebug("dnode rpc client is initialized"); return 0; } +int32_t dmInitStatusClient(SDnode *pDnode) { + SDnodeTrans *pTrans = &pDnode->trans; + + SRpcInit rpcInit = {0}; + rpcInit.label = "DND-STATUS-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = TSDB_DEFAULT_USER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.parent = pDnode; + rpcInit.rfp = rpcRfp; + rpcInit.compressSize = tsCompressMsgSize; + + rpcInit.retryMinInterval = tsRedirectPeriod; + rpcInit.retryStepFactor = tsRedirectFactor; + rpcInit.retryMaxInterval = tsRedirectMaxPeriod; + rpcInit.retryMaxTimeout = tsMaxRetryWaitTime; + + rpcInit.failFastInterval = 5000; // interval threshold(ms) + rpcInit.failFastThreshold = 3; // failed threshold + rpcInit.ffp = dmFailFastFp; + + int32_t connLimitNum = 100; + connLimitNum = TMAX(connLimitNum, 10); + connLimitNum = TMIN(connLimitNum, 500); + + rpcInit.connLimitNum = connLimitNum; + rpcInit.connLimitLock = 1; + rpcInit.supportBatch = 1; + rpcInit.batchSize = 8 * 1024; + rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + + pTrans->statusClientRpc = rpcOpen(&rpcInit); + if (pTrans->statusClientRpc == NULL) { + dError("failed to init dnode rpc status client"); + return -1; + } + + dDebug("dnode rpc status client is initialized"); + return 0; +} void dmCleanupClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -367,6 +411,14 @@ void dmCleanupClient(SDnode *pDnode) { dDebug("dnode rpc client is closed"); } } +void dmCleanupStatusClient(SDnode *pDnode) { + SDnodeTrans *pTrans = &pDnode->trans; + if (pTrans->statusClientRpc) { + rpcClose(pTrans->statusClientRpc); + pTrans->statusClientRpc = NULL; + dDebug("dnode rpc status client is closed"); + } +} int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -405,6 +457,7 @@ void dmCleanupServer(SDnode *pDnode) { SMsgCb dmGetMsgcb(SDnode *pDnode) { SMsgCb msgCb = { .clientRpc = pDnode->trans.clientRpc, + .statusClientRpc = pDnode->trans.statusClientRpc, .serverRpc = pDnode->trans.serverRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, From ef27c877565a8c3baec6d4aff72f77f4520be5d6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Oct 2023 16:37:34 +0800 Subject: [PATCH 071/177] opt status send --- include/util/tdef.h | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/libs/transport/src/trans.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 287617970c..7f8fe22340 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -249,7 +249,7 @@ typedef enum ELogicConditionType { #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 -#define TSDB_LABEL_LEN 8 +#define TSDB_LABEL_LEN 12 #define TSDB_JOB_STATUS_LEN 32 #define TSDB_CLUSTER_ID_LEN 40 diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index b273706feb..dbd68a2fbe 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -362,7 +362,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-STATUS-C"; + rpcInit.label = "DND-STATUS"; rpcInit.numOfThreads = 1; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index e9aa62eded..7b1ae087f2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -41,7 +41,8 @@ void* rpcOpen(const SRpcInit* pInit) { return NULL; } if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label); + tstrncpy(pRpc->label, pInit->label, len); } pRpc->compressSize = pInit->compressSize; From 5b69b584c3873af092d2737adff576a53874d315 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 18 Oct 2023 16:58:58 +0800 Subject: [PATCH 072/177] enh: add log for deploy mnode --- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 7840528db9..d25c6438e8 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -32,8 +32,13 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { if (!option.deploy) { *required = mmDeployRequired(pInput); + if (*required) { + dInfo("deploy mnode required. dnodeId:%d<=0, clusterId:%" PRId64 "<=0, localEp:%s==firstEp", + pInput->pData->dnodeId, pInput->pData->clusterId, tsLocalEp); + } } else { *required = true; + dInfo("deploy mnode required. option deploy:%d", option.deploy); } return 0; From 52c8bc91f98ebfea693b1b7c352eba2fdaf44a7c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 18 Oct 2023 18:21:45 +0800 Subject: [PATCH 073/177] cos/delete: fix delete object cond and double free --- source/dnode/vnode/src/vnd/vnodeCos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index c50f133987..ccda20e049 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -644,7 +644,7 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int } static void s3FreeObjectKey(void *pItem) { - char *key = (char *)pItem; + char *key = *(char **)pItem; taosMemoryFree(key); } @@ -682,7 +682,7 @@ void s3DeleteObjectsByPrefix(const char *prefix) { } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); if (data.status == S3StatusOK) { - if (!data.keyCount) { + if (data.keyCount > 0) { // printListBucketHeader(allDetails); s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); } From 788f21fd946afc1a9332d250305d8faac28941fa Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 18 Oct 2023 19:38:24 +0800 Subject: [PATCH 074/177] test: support replica3 in splited/transform tmqvnode --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 426646426d..c749864eec 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -173,6 +173,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 From 5852a0975e6c96bd5523511e80dffe3d7b1f56d5 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Mon, 16 Oct 2023 13:53:47 +0800 Subject: [PATCH 075/177] set fill history range --- source/libs/executor/inc/executorInt.h | 2 ++ source/libs/executor/src/filloperator.c | 13 +++++++++++-- tests/script/tsim/stream/ignoreCheckUpdate.sim | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 69330a8aee..288919d709 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -636,6 +636,7 @@ typedef struct SStreamFillSupporter { SSHashObj* pResMap; bool hasDelete; SStorageAPI* pAPI; + STimeWindow winRange; } SStreamFillSupporter; typedef struct SStreamFillOperatorInfo { @@ -770,6 +771,7 @@ SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo); bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType); bool compareVal(const char* v, const SStateKeys* pKey); +bool inWinRange(STimeWindow* range, STimeWindow* cur); int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, int32_t prevPosition, int32_t order); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 9fce058c4c..ecc2526dd9 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -950,7 +950,10 @@ static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if (inWinRange(&pFillSup->winRange, &st)) { + buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); } @@ -960,7 +963,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { uint64_t groupId = pBlock->info.id.groupId; SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; - if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) { + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); pFillInfo->pLinearInfo->winIndex++; @@ -1345,6 +1349,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { pInfo->pFillInfo->preRowKey = INT64_MIN; } + pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pInfo->pFillSup->winRange.ekey <= 0) { + pInfo->pFillSup->winRange.ekey = INT64_MAX; + } + switch (pBlock->info.type) { case STREAM_RETRIEVE: return pBlock; diff --git a/tests/script/tsim/stream/ignoreCheckUpdate.sim b/tests/script/tsim/stream/ignoreCheckUpdate.sim index 725251b081..108c845e4d 100644 --- a/tests/script/tsim/stream/ignoreCheckUpdate.sim +++ b/tests/script/tsim/stream/ignoreCheckUpdate.sim @@ -206,6 +206,8 @@ print create stream streams3 trigger at_once ignore update 1 into streamt3 as se sql create stream streams3 trigger at_once ignore update 1 into streamt3 as select _wstart c1, count(*) c2, max(b) c3 from st interval(10s); +sleep 2000 + sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,2); From 1910540a611c92b1dbba152954f2876af498147e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 10:15:04 +0800 Subject: [PATCH 076/177] add rpc sync read timeout --- source/libs/transport/inc/transComm.h | 23 ++++-- source/libs/transport/src/trans.c | 3 + source/libs/transport/src/transCli.c | 100 +++++++++++++++++++++++--- source/libs/transport/src/transComm.c | 21 +++++- 4 files changed, 131 insertions(+), 16 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index c82b7c0532..73427446e6 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -119,6 +119,13 @@ typedef struct SExHandle { void* pThrd; } SExHandle; +typedef struct { + STransMsg* pRsp; + tsem_t* pSem; + int8_t inited; + SRWLatch latch; +} STransSyncMsg; + /*convet from fqdn to ip */ typedef struct SCvtAddr { char ip[TSDB_FQDN_LEN]; @@ -133,11 +140,13 @@ typedef struct { tmsg_t msgType; // message type int8_t connType; // connection type cli/srv - STransCtx appCtx; // - STransMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - SCvtAddr cvtAddr; - bool setMaxRetry; + STransCtx appCtx; // + STransMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + STransSyncMsg* pSyncMsg; // for syncchronous with timeout API + int64_t syncMsgRef; + SCvtAddr cvtAddr; + bool setMaxRetry; int32_t retryMinInterval; int32_t retryMaxInterval; @@ -307,6 +316,7 @@ int transReleaseSrvHandle(void* handle); int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); +int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int32_t timeoutMs); int transSendResponse(const STransMsg* msg); int transRegisterMsg(const STransMsg* msg); int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); @@ -432,10 +442,11 @@ int64_t transAddExHandle(int32_t refMgt, void* p); int32_t transRemoveExHandle(int32_t refMgt, int64_t refId); void* transAcquireExHandle(int32_t refMgt, int64_t refId); int32_t transReleaseExHandle(int32_t refMgt, int64_t refId); -void transDestoryExHandle(void* handle); +void transDestroyExHandle(void* handle); int32_t transGetRefMgt(); int32_t transGetInstMgt(); +int32_t transGetSyncMsgMgt(); void transHttpEnvDestroy(); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 7b1ae087f2..6842d9ee82 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -169,6 +169,9 @@ int rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, in int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { return transSendRecv(shandle, pEpSet, pMsg, pRsp); } +int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, timeoutMs); +} int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 677e08ec56..3156a0733c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2411,15 +2411,24 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } } } - if (pCtx->pSem != NULL) { + if (pCtx->pSem || pCtx->syncMsgRef != 0) { tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); - if (pCtx->pRsp == NULL) { - tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); + if (pCtx->pSem) { + if (pCtx->pRsp == NULL) { + tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); + } else { + memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + } + tsem_post(pCtx->pSem); + pCtx->pRsp = NULL; } else { - memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + STransSyncMsg* pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); + if (pSyncMsg != NULL) { + memcpy(pSyncMsg->pRsp, (char*)pResp, sizeof(*pResp)); + tsem_post(pSyncMsg->pSem); + taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); + } } - tsem_post(pCtx->pSem); - pCtx->pRsp = NULL; } else { tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (retry == false && hasEpSet == true) { @@ -2563,7 +2572,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran } int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + STransMsg* pTransRsp = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); return -1; @@ -2587,7 +2597,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; pCtx->pSem = sem; - pCtx->pRsp = pRsp; + pCtx->pRsp = pTransRsp; SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; @@ -2607,10 +2617,84 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs } tsem_wait(sem); + memcpy(pRsp, pTransRsp, sizeof(STransMsg)); + _RETURN: tsem_destroy(sem); taosMemoryFree(sem); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + taosMemoryFree(pTransRsp); + return ret; +} +int64_t transCreateSyncMsg(STransMsg* pTransMsg) { + tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); + tsem_init(sem, 0, 0); + + STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg)); + + taosInitRWLatch(&pSyncMsg->latch); + pSyncMsg->inited = 0; + pSyncMsg->pRsp = pTransMsg; + pSyncMsg->pSem = sem; + + return taosAddRef(transGetSyncMsgMgt(), pSyncMsg); +} +int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int32_t timeoutMs) { + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return -1; + } + + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return TSDB_CODE_RPC_BROKEN_LINK; + } + + TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); + epsetAssign(&pCtx->epSet, pEpSet); + epsetAssign(&pCtx->origEpSet, pEpSet); + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + pCtx->syncMsgRef = transCreateSyncMsg(pTransMsg); + + int64_t ref = pCtx->syncMsgRef; + STransSyncMsg* pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), ref); + + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); + cliMsg->ctx = pCtx; + cliMsg->msg = *pReq; + cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; + cliMsg->refId = (int64_t)shandle; + + STraceId* trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); + + int ret = transAsyncSend(pThrd->asyncPool, &cliMsg->q); + if (ret != 0) { + destroyCmsg(cliMsg); + goto _RETURN; + } + + ret = tsem_timewait(pSyncMsg->pSem, timeoutMs); + if (ret < 0) { + pRsp->code = TSDB_CODE_TIMEOUT_ERROR; + ret = TSDB_CODE_TIMEOUT_ERROR; + } else { + memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + ret = 0; + } +_RETURN: + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + taosReleaseRef(transGetSyncMsgMgt(), ref); + taosRemoveRef(transGetSyncMsgMgt(), ref); return ret; } /* diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 3dc59a93ee..759a4d79db 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -21,6 +21,9 @@ static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; static int32_t refMgt; static int32_t instMgt; +static int32_t transSyncMsgMgt; + +void transDestroySyncMsg(void* msg); int32_t transCompressMsg(char* msg, int32_t len) { int32_t ret = 0; @@ -601,13 +604,15 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { } static void transInitEnv() { - refMgt = transOpenRefMgt(50000, transDestoryExHandle); + refMgt = transOpenRefMgt(50000, transDestroyExHandle); instMgt = taosOpenRef(50, rpcCloseImpl); + transSyncMsgMgt = taosOpenRef(50, transDestroySyncMsg); uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1"); } static void transDestroyEnv() { transCloseRefMgt(refMgt); transCloseRefMgt(instMgt); + transCloseRefMgt(transSyncMsgMgt); } void transInit() { @@ -617,6 +622,7 @@ void transInit() { int32_t transGetRefMgt() { return refMgt; } int32_t transGetInstMgt() { return instMgt; } +int32_t transGetSyncMsgMgt() { return transSyncMsgMgt; } void transCleanup() { // clean env @@ -648,13 +654,24 @@ int32_t transReleaseExHandle(int32_t refMgt, int64_t refId) { // release extern handle return taosReleaseRef(refMgt, refId); } -void transDestoryExHandle(void* handle) { +void transDestroyExHandle(void* handle) { if (handle == NULL) { return; } taosMemoryFree(handle); } +void transDestroySyncMsg(void* msg) { + if (msg == NULL) return; + + STransSyncMsg* pSyncMsg = msg; + tsem_destroy(pSyncMsg->pSem); + taosMemoryFree(pSyncMsg->pSem); + + taosMemoryFree(pSyncMsg->pRsp); + taosMemoryFree(pSyncMsg); +} + // void subnetIp2int(const char* const ip_addr, uint8_t* dst) { // char ip_addr_cpy[20]; // char ip[5]; From 573c28fd067364d1db2b3ff36bad58735d88caf5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 23 Aug 2023 18:37:29 +0800 Subject: [PATCH 077/177] enh: iterate over a list of snap ranges for STsdbSnapReader --- source/dnode/vnode/src/inc/tsdb.h | 3 ++ source/dnode/vnode/src/tsdb/tsdbFS2.c | 34 ++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 5 ++- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 21 ++++++++++- source/dnode/vnode/src/tsdb/tsdbFSet2.h | 13 ++++++- source/dnode/vnode/src/tsdb/tsdbFile2.h | 4 ++- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 41 ++++++++++------------ 7 files changed, 95 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e83f47f7b6..83c14a50d0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -672,6 +672,9 @@ struct SDelFWriter { typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; +typedef struct STSnapRange STSnapRange; +typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges + struct STsdbReadSnap { SMemTable *pMem; SQueryNode *pNode; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index afe6ef6e1a..cae6f06636 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -991,6 +991,40 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, TSnapRangeArray **fsrArr) { + int32_t code = 0; + STFileSet *fset; + STSnapRange *fsr1; + + fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); + if (fsrArr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTSnapRangeInitRef(fs->tsdb, fset, &fsr1); + if (code) break; + + code = TARRAY2_APPEND(fsrArr[0], fsr1); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); + fsrArr[0] = NULL; + } + return code; +} + +int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr) { + if (fsrArr[0]) { + TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); + taosMemoryFreeClear(fsrArr[0]); + fsrArr[0] = NULL; + } + return 0; +} + const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; static int32_t tsdbFSRunBgTask(void *arg) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index b0f42a0c48..4b7e0bba2c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -52,7 +52,10 @@ int32_t tsdbCloseFS(STFileSystem **fs); int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); -int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); +int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsrArr); + +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, TSnapRangeArray **fsrArr); +int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index cd47a54973..67b6d8f003 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -458,6 +458,16 @@ int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, STSnapRange **fsr) { + fsr[0] = taosMemoryCalloc(1, sizeof(STSnapRange)); + if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + fsr[0]->fid = fset1->fid; + // fsr[0]->sver = sver; + // fsr[0]->ever = ever; + return tsdbTFileSetInitRef(pTsdb, fset1, &fsr[0]->fset); +} + int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { int32_t code = tsdbTFileSetInit(fset1->fid, fset); if (code) return code; @@ -485,6 +495,15 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } +int32_t tsdbTSnapRangeClear(STSnapRange **fsr) { + if (!fsr[0]) return 0; + + tsdbTFileSetClear(&fsr[0]->fset); + taosMemoryFree(fsr[0]); + fsr[0] = NULL; + return 0; +} + int32_t tsdbTFileSetClear(STFileSet **fset) { if (!fset[0]) return 0; @@ -545,4 +564,4 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) { if (fset->farr[ftype] != NULL) return false; } return TARRAY2_SIZE(fset->lvlArr) == 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index d7b3c1fc8c..da5c4cbcc3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -45,6 +45,10 @@ int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); int32_t tsdbTFileSetClear(STFileSet **fset); int32_t tsdbTFileSetRemove(STFileSet **fset); + +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, STSnapRange **fsr); +int32_t tsdbTSnapRangeClear(STSnapRange **fsr); + // to/from json int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset); @@ -78,8 +82,15 @@ struct STFileSet { TSttLvlArray lvlArr[1]; // level array }; +struct STSnapRange { + int32_t fid; + int64_t sver; + int64_t ever; + STFileSet *fset; +}; + #ifdef __cplusplus } #endif -#endif /*_TSDB_FILE_SET2_H*/ \ No newline at end of file +#endif /*_TSDB_FILE_SET2_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h index 11d08e45e6..33d8ac5478 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h @@ -61,6 +61,8 @@ struct STFile { int32_t fid; // file id int64_t cid; // commit id int64_t size; + int64_t minVer; + int64_t maxVer; union { struct { int32_t level; @@ -80,4 +82,4 @@ struct STFileObj { } #endif -#endif /*_TSDB_FILE_H*/ \ No newline at end of file +#endif /*_TSDB_FILE_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index e4011ca400..e016654b4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -32,12 +32,12 @@ struct STsdbSnapReader { uint8_t* aBuf[5]; SSkmInfo skmTb[1]; - TFileSetArray* fsetArr; + TSnapRangeArray* fsrArr; // context struct { - int32_t fsetArrIdx; - STFileSet* fset; + int32_t fsrArrIdx; + STSnapRange* fsr; bool isDataDone; bool isTombDone; } ctx[1]; @@ -72,10 +72,10 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { }; bool hasDataFile = false; for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { - if (reader->ctx->fset->farr[ftype] != NULL) { + if (reader->ctx->fsr->fset->farr[ftype] != NULL) { hasDataFile = true; config.files[ftype].exist = true; - config.files[ftype].file = reader->ctx->fset->farr[ftype]->f[0]; + config.files[ftype].file = reader->ctx->fsr->fset->farr[ftype]->f[0]; } } @@ -86,7 +86,7 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { // stt SSttLvl* lvl; - TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(reader->ctx->fsr->fset->lvlArr, lvl) { STFileObj* fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { SSttFileReader* sttReader; @@ -211,14 +211,14 @@ static int32_t tsdbSnapReadFileSetCloseIter(STsdbSnapReader* reader) { return 0; } -static int32_t tsdbSnapReadFileSetBegin(STsdbSnapReader* reader) { +static int32_t tsdbSnapReadRangeBegin(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - ASSERT(reader->ctx->fset == NULL); + ASSERT(reader->ctx->fsr == NULL); - if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) { - reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++); + if (reader->ctx->fsrArrIdx < TARRAY2_SIZE(reader->fsrArr)) { + reader->ctx->fsr = TARRAY2_GET(reader->fsrArr, reader->ctx->fsrArrIdx++); reader->ctx->isDataDone = false; reader->ctx->isTombDone = false; @@ -236,10 +236,10 @@ _exit: return code; } -static int32_t tsdbSnapReadFileSetEnd(STsdbSnapReader* reader) { +static int32_t tsdbSnapReadRangeEnd(STsdbSnapReader* reader) { tsdbSnapReadFileSetCloseIter(reader); tsdbSnapReadFileSetCloseReader(reader); - reader->ctx->fset = NULL; + reader->ctx->fsr = NULL; return 0; } @@ -424,17 +424,14 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - taosThreadRwlockRdlock(&tsdb->rwLock); - code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr); - taosThreadRwlockUnlock(&tsdb->rwLock); - + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -462,7 +459,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -488,11 +485,11 @@ int32_t tsdbSnapRead(STsdbSnapReader* reader, uint8_t** data) { data[0] = NULL; for (;;) { - if (reader->ctx->fset == NULL) { - code = tsdbSnapReadFileSetBegin(reader); + if (reader->ctx->fsr == NULL) { + code = tsdbSnapReadRangeBegin(reader); TSDB_CHECK_CODE(code, lino, _exit); - if (reader->ctx->fset == NULL) { + if (reader->ctx->fsr == NULL) { break; } } @@ -517,7 +514,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* reader, uint8_t** data) { } } - code = tsdbSnapReadFileSetEnd(reader); + code = tsdbSnapReadRangeEnd(reader); TSDB_CHECK_CODE(code, lino, _exit); } From 1be2835fc1c3e0e71d226829b21c58697a79a2c8 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 24 Aug 2023 16:17:50 +0800 Subject: [PATCH 078/177] enh: exclude a list of snap ranges in tsdbSnapReaderOpen --- source/dnode/vnode/src/inc/vnodeInt.h | 3 ++- source/dnode/vnode/src/sma/smaSnapshot.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 10 +++++++--- source/dnode/vnode/src/tsdb/tsdbFS2.h | 3 ++- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 17 +++++++++++------ source/dnode/vnode/src/tsdb/tsdbFSet2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 6 +++++- 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 823e9d57f6..cc355c5f32 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -295,7 +295,8 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback); // STsdbSnapReader ======================================== -int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader); +int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pEx, + STsdbSnapReader** ppReader); int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader); int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData); // STsdbSnapWriter ======================================== diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index e01a33936b..2a010f5a84 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -48,7 +48,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead // open rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2, + code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, (i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2), NULL, &pReader->pDataReader[i]); TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index cae6f06636..0a8bb4c37a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -991,25 +991,29 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, TSnapRangeArray **fsrArr) { +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pEx, + TSnapRangeArray **fsrArr) { int32_t code = 0; STFileSet *fset; - STSnapRange *fsr1; + STSnapRange *fsr1 = NULL; fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); if (fsrArr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; taosThreadRwlockRdlock(&fs->tsdb->rwLock); TARRAY2_FOREACH(fs->fSetArr, fset) { - code = tsdbTSnapRangeInitRef(fs->tsdb, fset, &fsr1); + code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver, ever, &fsr1); if (code) break; code = TARRAY2_APPEND(fsrArr[0], fsr1); if (code) break; + + fsr1 = NULL; } taosThreadRwlockUnlock(&fs->tsdb->rwLock); if (code) { + tsdbTSnapRangeClear(&fsr1); TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); fsrArr[0] = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 4b7e0bba2c..8dfc86ee83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -54,7 +54,8 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsrArr); -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, TSnapRangeArray **fsrArr); +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pEx, + TSnapRangeArray **fsrArr); int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 67b6d8f003..e0434b7da6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -458,14 +458,19 @@ int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, STSnapRange **fsr) { - fsr[0] = taosMemoryCalloc(1, sizeof(STSnapRange)); +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) { + fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; - fsr[0]->fid = fset1->fid; - // fsr[0]->sver = sver; - // fsr[0]->ever = ever; - return tsdbTFileSetInitRef(pTsdb, fset1, &fsr[0]->fset); + fsr[0]->sver = sver; + fsr[0]->ever = ever; + + int32_t code = tsdbTFileSetInitRef(pTsdb, fset1, &fsr[0]->fset); + if (code) { + taosMemoryFree(fsr[0]); + fsr[0] = NULL; + } + return code; } int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index da5c4cbcc3..8155328e70 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -46,7 +46,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs int32_t tsdbTFileSetClear(STFileSet **fset); int32_t tsdbTFileSetRemove(STFileSet **fset); -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, STSnapRange **fsr); +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr); int32_t tsdbTSnapRangeClear(STSnapRange **fsr); // to/from json diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index e016654b4b..618e10195b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -412,7 +412,7 @@ _exit: return code; } -int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** reader) { +int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, void* pEx, STsdbSnapReader** reader) { int32_t code = 0; int32_t lino = 0; @@ -424,7 +424,7 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, &reader[0]->fsrArr); + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pEx, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 3abcf79839..b89fed73f8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -14,6 +14,7 @@ */ #include "vnd.h" +#include "tsdb.h" // SVSnapReader ======================================================== struct SVSnapReader { @@ -28,6 +29,7 @@ struct SVSnapReader { SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; + TSnapRangeArray *pEx; STsdbSnapReader *pTsdbReader; // tq int8_t tqHandleDone; @@ -59,6 +61,8 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe pReader->sver = sver; pReader->ever = ever; + // TODO: pReader->pEx + vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; return code; @@ -175,7 +179,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { - code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, + code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pEx, &pReader->pTsdbReader); if (code) goto _err; } From 7a13e4b2cf0c454cd92c635b96ecdafb67771ddd Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 25 Aug 2023 11:50:36 +0800 Subject: [PATCH 079/177] enh: add minVer and maxVer for entries in current.json --- source/dnode/vnode/src/tsdb/tsdbFile2.c | 26 ++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 3d8964d41b..963c5bad34 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -76,6 +76,17 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) { return TSDB_CODE_OUT_OF_MEMORY; } + if (file->minVer <= file->maxVer) { + /* minVer */ + if (cJSON_AddNumberToObject(json, "minVer", file->minVer) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* maxVer */ + if (cJSON_AddNumberToObject(json, "maxVer", file->maxVer) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } return 0; } @@ -122,6 +133,19 @@ static int32_t tfile_from_json(const cJSON *json, STFile *file) { return TSDB_CODE_FILE_CORRUPTED; } + /* minVer */ + file->minVer = VERSION_MAX; + item = cJSON_GetObjectItem(json, "minVer"); + if (cJSON_IsNumber(item)) { + file->minVer = item->valuedouble; + } + + /* maxVer */ + file->maxVer = VERSION_MIN; + item = cJSON_GetObjectItem(json, "maxVer"); + if (cJSON_IsNumber(item)) { + file->maxVer = item->valuedouble; + } return 0; } @@ -296,4 +320,4 @@ int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) { } else { return 0; } -} \ No newline at end of file +} From da28d490aab664086feaafcf88b2e0428f21b6ca Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 25 Aug 2023 11:59:59 +0800 Subject: [PATCH 080/177] enh: filterByVersion with snap ranges --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 618e10195b..b604256e89 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -138,8 +138,8 @@ static int32_t tsdbSnapReadFileSetOpenIter(STsdbSnapReader* reader) { STsdbIter* iter; STsdbIterConfig config = { .filterByVersion = true, - .verRange[0] = reader->sver, - .verRange[1] = reader->ever, + .verRange[0] = reader->ctx->fsr->sver, + .verRange[1] = reader->ctx->fsr->ever, }; // data file From ab7a20c1171a847da18c4e9bef8b030c83bfa9ae Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 25 Aug 2023 19:04:10 +0800 Subject: [PATCH 081/177] enh: write stt and data files with minVer and maxVer --- source/dnode/vnode/src/inc/vnodeInt.h | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbCommit2.c | 8 ++++++++ source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 8 ++++++++ source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 4 +++- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 6 +++++- source/dnode/vnode/src/tsdb/tsdbFSetRW.h | 4 +++- source/dnode/vnode/src/tsdb/tsdbMerge.c | 1 + source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbSttFileRW.h | 4 +++- source/dnode/vnode/src/vnd/vnodeCommit.c | 3 +++ 10 files changed, 42 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index cc355c5f32..7a242b55cf 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -527,10 +527,16 @@ struct SSnapDataHdr { uint8_t data[]; }; +typedef struct SRange { + int64_t start; + int64_t end; +} SRange; + struct SCommitInfo { SVnodeInfo info; SVnode* pVnode; TXN* txn; + SRange vers; }; struct SCompactInfo { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 79964c5636..4e096a7f17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -43,6 +43,8 @@ typedef struct { SDiskID did; TSKEY minKey; TSKEY maxKey; + int64_t minVer; + int64_t maxVer; STFileSet *fset; TABLEID tbid[1]; bool hasTSData; @@ -74,6 +76,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { .szPage = committer->szPage, .cmprAlg = committer->cmprAlg, .fid = committer->ctx->fid, + .minVer = committer->ctx->minVer, + .maxVer = committer->ctx->maxVer, .cid = committer->ctx->cid, .did = committer->ctx->did, .level = 0, @@ -87,6 +91,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { if (committer->ctx->fset->farr[ftype] != NULL) { config.files[ftype].exist = true; config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0]; + config.files[ftype].file.minVer = TMIN(config.files[ftype].file.minVer, config.minVer); + config.files[ftype].file.maxVer = TMAX(config.files[ftype].file.maxVer, config.maxVer); } } } @@ -463,6 +469,8 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->compactVersion = INT64_MAX; committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); committer->ctx->now = taosGetTimestampSec(); + committer->ctx->minVer = info->vers.start; + committer->ctx->maxVer = info->vers.end; committer->ctx->nextKey = tsdb->imem->minKey; if (tsdb->imem->nDel > 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 6e4cb517ff..3265bb7cc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -589,6 +589,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, + .minVer = writer->config->minVer, + .maxVer = writer->config->maxVer, }; // .data @@ -602,6 +604,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, + .minVer = writer->config->minVer, + .maxVer = writer->config->maxVer, }; } @@ -616,6 +620,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, + .minVer = writer->config->minVer, + .maxVer = writer->config->maxVer, }; } @@ -627,6 +633,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, + .minVer = writer->config->minVer, + .maxVer = writer->config->maxVer, }; writer->ctx->opened = true; diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index 827b58fb4a..e87c00d382 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -76,6 +76,8 @@ typedef struct SDataFileWriterConfig { int32_t maxRow; int32_t szPage; int32_t fid; + int64_t minVer; + int64_t maxVer; int64_t cid; SDiskID did; int64_t compactVersion; @@ -101,4 +103,4 @@ int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord * } #endif -#endif /*_TSDB_DATA_FILE_RW_H*/ \ No newline at end of file +#endif /*_TSDB_DATA_FILE_RW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index 83ae8c2429..801ae59838 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -143,6 +143,8 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .maxRow = config->maxRow, .szPage = config->szPage, .fid = config->fid, + .minVer = config->minVer, + .maxVer = config->maxVer, .cid = config->cid, .did = config->did, .compactVersion = config->compactVersion, @@ -168,6 +170,8 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .compactVersion = config->compactVersion, .did = config->did, .fid = config->fid, + .minVer = config->minVer, + .maxVer = config->maxVer, .cid = config->cid, .level = config->level, .skmTb = writer[0]->skmTb, @@ -292,4 +296,4 @@ _exit: TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); } return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h index b5710407cf..a733bb3c44 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h @@ -34,6 +34,8 @@ typedef struct { int32_t szPage; int8_t cmprAlg; int32_t fid; + int64_t minVer; + int64_t maxVer; int64_t cid; SDiskID did; int32_t level; @@ -52,4 +54,4 @@ int32_t tsdbFSetWriteTombRecord(SFSetWriter *writer, const STombRecord *tombReco } #endif -#endif /*_TSDB_FSET_RW_H*/ \ No newline at end of file +#endif /*_TSDB_FSET_RW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 42a8b5bb3f..e659cedba3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -313,6 +313,7 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) { if (merger->ctx->fset->farr[ftype]) { config.files[ftype].exist = true; config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0]; + } else { config.files[ftype].exist = false; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index fa8d2d5ba4..4f1eb49959 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -694,6 +694,8 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, + .minVer = writer->config->minVer, + .maxVer = writer->config->maxVer, .stt[0] = { .level = writer->config->level, diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h index 242b55795c..d0481d5ec3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h @@ -79,6 +79,8 @@ struct SSttFileWriterConfig { int64_t compactVersion; SDiskID did; int32_t fid; + int64_t minVer; + int64_t maxVer; int64_t cid; int32_t level; SSkmInfo *skmTb; @@ -90,4 +92,4 @@ struct SSttFileWriterConfig { } #endif -#endif /*_TSDB_STT_FILE_RW_H*/ \ No newline at end of file +#endif /*_TSDB_STT_FILE_RW_H*/ diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 136168c5cc..775b298268 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -285,6 +285,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { int32_t code = 0; int32_t lino = 0; char dir[TSDB_FILENAME_LEN] = {0}; + int64_t lastCommitted = pInfo->info.state.committed; tsem_wait(&pVnode->canCommit); @@ -296,6 +297,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { pInfo->info.state.committed = pVnode->state.applied; pInfo->info.state.commitTerm = pVnode->state.applyTerm; pInfo->info.state.commitID = ++pVnode->state.commitID; + pInfo->vers.start = lastCommitted + 1; + pInfo->vers.end = pInfo->info.state.committed; pInfo->pVnode = pVnode; pInfo->txn = metaGetTxn(pVnode->pMeta); From c8a46394f1774da2ba857d46279ade0f62924798 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 25 Aug 2023 19:05:03 +0800 Subject: [PATCH 082/177] enh: set default values of minVer and maxVer while upgrading from CURRENT --- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 0884c32385..225822ed97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -78,6 +78,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * .fid = fset->fid, .cid = pDFileSet->pHeadF->commitID, .size = pDFileSet->pHeadF->size, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]); @@ -182,6 +184,8 @@ static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * .fid = fset->fid, .cid = pDFileSet->pDataF->commitID, .size = pDFileSet->pDataF->size, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]); @@ -208,6 +212,8 @@ static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r .fid = fset->fid, .cid = pDFileSet->pSmaF->commitID, .size = pDFileSet->pSmaF->size, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]); @@ -253,6 +259,8 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade .fid = fset->fid, .cid = pSttF->commitID, .size = pSttF->size, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; code = tsdbTFileObjInit(tsdb, &file, &fobj); TSDB_CHECK_CODE(code, lino, _exit1); @@ -382,6 +390,8 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f .fid = fset->fid, .cid = 0, .size = 0, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; code = tsdbTFileObjInit(tsdb, &file, fobj); @@ -398,6 +408,8 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f .fid = fset->fid, .cid = 0, .size = 0, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; code = tsdbTFileObjInit(tsdb, &file, fobj); From 1446b4473328934d2f5967a03328c6c4fa573a0e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 10:25:53 +0800 Subject: [PATCH 083/177] fixup: remove minVer and maxVer in FSetWriterConfig --- source/dnode/vnode/src/inc/vnodeInt.h | 6 ------ source/dnode/vnode/src/tsdb/tsdbCommit2.c | 6 ------ source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbFSetRW.h | 2 -- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 -- 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7a242b55cf..cc355c5f32 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -527,16 +527,10 @@ struct SSnapDataHdr { uint8_t data[]; }; -typedef struct SRange { - int64_t start; - int64_t end; -} SRange; - struct SCommitInfo { SVnodeInfo info; SVnode* pVnode; TXN* txn; - SRange vers; }; struct SCompactInfo { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 4e096a7f17..302991fb0f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -76,8 +76,6 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { .szPage = committer->szPage, .cmprAlg = committer->cmprAlg, .fid = committer->ctx->fid, - .minVer = committer->ctx->minVer, - .maxVer = committer->ctx->maxVer, .cid = committer->ctx->cid, .did = committer->ctx->did, .level = 0, @@ -91,8 +89,6 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { if (committer->ctx->fset->farr[ftype] != NULL) { config.files[ftype].exist = true; config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0]; - config.files[ftype].file.minVer = TMIN(config.files[ftype].file.minVer, config.minVer); - config.files[ftype].file.maxVer = TMAX(config.files[ftype].file.maxVer, config.maxVer); } } } @@ -469,8 +465,6 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->compactVersion = INT64_MAX; committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); committer->ctx->now = taosGetTimestampSec(); - committer->ctx->minVer = info->vers.start; - committer->ctx->maxVer = info->vers.end; committer->ctx->nextKey = tsdb->imem->minKey; if (tsdb->imem->nDel > 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index 801ae59838..4397d1cb5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -143,8 +143,8 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .maxRow = config->maxRow, .szPage = config->szPage, .fid = config->fid, - .minVer = config->minVer, - .maxVer = config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, .cid = config->cid, .did = config->did, .compactVersion = config->compactVersion, @@ -170,8 +170,8 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .compactVersion = config->compactVersion, .did = config->did, .fid = config->fid, - .minVer = config->minVer, - .maxVer = config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, .cid = config->cid, .level = config->level, .skmTb = writer[0]->skmTb, diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h index a733bb3c44..0a8049cded 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h @@ -34,8 +34,6 @@ typedef struct { int32_t szPage; int8_t cmprAlg; int32_t fid; - int64_t minVer; - int64_t maxVer; int64_t cid; SDiskID did; int32_t level; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 775b298268..50ca2f5d03 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -297,8 +297,6 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { pInfo->info.state.committed = pVnode->state.applied; pInfo->info.state.commitTerm = pVnode->state.applyTerm; pInfo->info.state.commitID = ++pVnode->state.commitID; - pInfo->vers.start = lastCommitted + 1; - pInfo->vers.end = pInfo->info.state.committed; pInfo->pVnode = pVnode; pInfo->txn = metaGetTxn(pVnode->pMeta); From 3cd458f5c98a53e1616b4bba78210f6bfeab5425 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 11:13:03 +0800 Subject: [PATCH 084/177] enh: fill minVer and maxVer of nf in STFileObj during migration --- source/dnode/vnode/src/tsdb/tsdbRetention.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index c3b1a18fd8..f2665dcf26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -151,6 +151,8 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const .type = fobj->f->type, .did = did[0], .fid = fobj->f->fid, + .minVer = fobj->f->minVer, + .maxVer = fobj->f->maxVer, .cid = fobj->f->cid, .size = fobj->f->size, .stt[0] = @@ -198,6 +200,8 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const .type = fobj->f->type, .did = did[0], .fid = fobj->f->fid, + .minVer = fobj->f->minVer, + .maxVer = fobj->f->maxVer, .cid = fobj->f->cid, .size = fobj->f->size, .stt[0] = From 823aad4a5ec831854acf930017bc647a703bc819 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 14:06:09 +0800 Subject: [PATCH 085/177] enh: record version range of STFile for data and stt files --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 62 ++++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 32 ++++++++-- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 16 +++-- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 3265bb7cc7..64c9e9d517 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -16,7 +16,7 @@ #include "tsdbDataFileRW.h" extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr); + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); // SDataFileReader ============================================= @@ -589,8 +589,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, - .minVer = writer->config->minVer, - .maxVer = writer->config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; // .data @@ -604,8 +604,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, - .minVer = writer->config->minVer, - .maxVer = writer->config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; } @@ -620,8 +620,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, - .minVer = writer->config->minVer, - .maxVer = writer->config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; } @@ -633,8 +633,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, - .minVer = writer->config->minVer, - .maxVer = writer->config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, }; writer->ctx->opened = true; @@ -646,8 +646,14 @@ _exit: return code; } +static int32_t tsdbSDataUpdVerRange(SDataFileWriterConfig *config, SVersionRange *range) { + config->minVer = TMIN(config->minVer, range->minVer); + config->maxVer = TMAX(config->maxVer, range->maxVer); + return 0; +} + int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, - TBrinBlkArray *brinBlkArray, uint8_t **bufArr) { + TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range) { if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0; int32_t code; @@ -686,6 +692,9 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl } } + range->minVer = brinBlk->minVer; + range->maxVer = brinBlk->maxVer; + // write to file for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr1 + i), TARRAY2_DATA_LEN(brinBlock->dataArr1 + i), @@ -734,11 +743,15 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, - &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr); + &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr, + &range); TSDB_CHECK_CODE(code, lino, _exit); + tsdbSDataUpdVerRange(writer->config, &range); + _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); @@ -803,6 +816,9 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData } } + SVersionRange range = {.minVer = record->minVer, .maxVer = record->maxVer}; + tsdbSDataUpdVerRange(writer->config, &range); + // to .data file int32_t sizeArr[5] = {0}; @@ -1170,10 +1186,12 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - + + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, - &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr); + &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); + tsdbSDataUpdVerRange(writer->config, &range); _exit: if (code) { @@ -1358,6 +1376,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); @@ -1369,6 +1389,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { @@ -1378,6 +1400,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .of = writer->config->files[ftype].file, .nf = writer->files[ftype], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1390,6 +1414,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { @@ -1399,6 +1425,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .of = writer->config->files[ftype].file, .nf = writer->files[ftype], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1438,6 +1466,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1447,9 +1477,14 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr code = tsdbFsyncFile(writer->fd[i]); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&writer->fd[i]); + writer->files[i].minVer = TMIN(writer->files[i].minVer, writer->config->minVer); + writer->files[i].maxVer = TMAX(writer->files[i].maxVer, writer->config->maxVer); } } + writer->config->minVer = VERSION_MAX; + writer->config->maxVer = VERSION_MIN; + _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); @@ -1606,6 +1641,7 @@ int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { ) { code = tsdbDataFileDoWriteBlockData(writer, bData); TSDB_CHECK_CODE(code, lino, _exit); + } else { for (int32_t i = 0; i < bData->nRow; ++i) { TSDBROW row[1] = {tsdbRowFromBlockData(bData, i)}; diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 4f1eb49959..eaa5e40726 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -401,8 +401,14 @@ struct SSttFileWriter { uint8_t *bufArr[5]; }; -int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, - TSttBlkArray *sttBlkArray, uint8_t **bufArr) { +static int32_t tsdbSSttUpdVerRange(SSttFileWriterConfig *config, SVersionRange *range) { + config->minVer = TMIN(config->minVer, range->minVer); + config->maxVer = TMAX(config->maxVer, range->maxVer); + return 0; +} + +static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, + TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) { if (blockData->nRow == 0) return 0; int32_t code = 0; @@ -425,6 +431,9 @@ int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmpr if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow]; } + range->minVer = sttBlk->minVer; + range->maxVer = sttBlk->maxVer; + int32_t sizeArr[5] = {0}; code = tCmprBlockData(blockData, cmprAlg, NULL, NULL, bufArr, sizeArr); if (code) return code; @@ -455,9 +464,11 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - code = tsdbFileDoWriteBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, - writer->sttBlkArray, writer->config->bufArr); + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, + writer->sttBlkArray, writer->config->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); + tsdbSSttUpdVerRange(writer->config, &range); _exit: if (code) { @@ -518,7 +529,7 @@ _exit: } int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr) { + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range) { int32_t code; if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; @@ -554,6 +565,9 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl } } + range->minVer = tombBlk->minVer; + range->maxVer = tombBlk->maxVer; + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); @@ -579,9 +593,11 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, - writer->tombBlkArray, writer->config->bufArr); + writer->tombBlkArray, writer->config->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); + tsdbSSttUpdVerRange(writer->config, &range); _exit: if (code) { @@ -784,6 +800,10 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o .fid = writer->config->fid, .nf = writer->file[0], }; + op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + writer->config->minVer = VERSION_MAX; + writer->config->maxVer = VERSION_MIN; code = TARRAY2_APPEND(opArray, op); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 225822ed97..16f649cd9d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -23,7 +23,7 @@ extern int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SB extern int32_t save_fs(const TFileSetArray *arr, const char *fname); extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); extern int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, - TBrinBlkArray *brinBlkArray, uint8_t **bufArr); + TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range); extern int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); extern int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); extern int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); @@ -31,7 +31,7 @@ extern int32_t tsdbSttLvlClear(SSttLvl **lvl); extern int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); extern int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr); + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); extern int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); @@ -129,16 +129,18 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * TSDB_CHECK_CODE(code, lino, _exit); if (BRIN_BLOCK_SIZE(ctx->brinBlock) >= ctx->maxRow) { + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, - ctx->brinBlkArray, ctx->bufArr); + ctx->brinBlkArray, ctx->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); } } } if (BRIN_BLOCK_SIZE(ctx->brinBlock) > 0) { + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, - ctx->brinBlkArray, ctx->bufArr); + ctx->brinBlkArray, ctx->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); } @@ -493,8 +495,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TSDB_CHECK_CODE(code, lino, _exit); } + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->bufArr); + ctx->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -505,8 +508,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TSDB_CHECK_CODE(code, lino, _exit); } + SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->bufArr); + ctx->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); } From ed6d7c4c4d52f15262ec96cea820c01ed8c8fc48 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 14:59:07 +0800 Subject: [PATCH 086/177] refact: relocate declarations and defs of tsdbFileWriteTombBlock and tsdbFileWriteTombBlk --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 78 +++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 4 + source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 75 +------------------ source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 5 +- 4 files changed, 81 insertions(+), 81 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 64c9e9d517..2798eb0291 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -15,10 +15,6 @@ #include "tsdbDataFileRW.h" -extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); -extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); - // SDataFileReader ============================================= struct SDataFileReader { SDataFileReaderConfig config[1]; @@ -1167,6 +1163,65 @@ int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFoote return 0; } +int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range) { + int32_t code; + + if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; + + STombBlk tombBlk[1] = {{ + .dp[0] = + { + .offset = *fileSize, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(tombBlock->suid), + .uid = TARRAY2_FIRST(tombBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(tombBlock->suid), + .uid = TARRAY2_LAST(tombBlock->uid), + }, + .minVer = TARRAY2_FIRST(tombBlock->version), + .maxVer = TARRAY2_FIRST(tombBlock->version), + .numRec = TOMB_BLOCK_SIZE(tombBlock), + .cmprAlg = cmprAlg, + }}; + + for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) { + if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) { + tombBlk->minVer = TARRAY2_GET(tombBlock->version, i); + } + if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) { + tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i); + } + } + + range->minVer = tombBlk->minVer; + range->maxVer = tombBlk->maxVer; + + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), + TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]); + if (code) return code; + + tombBlk->dp->size += tombBlk->size[i]; + *fileSize += tombBlk->size[i]; + } + + code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk); + if (code) return code; + + tTombBlockClear(tombBlock); + return 0; +} + static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; @@ -1200,6 +1255,21 @@ _exit: return code; } +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(tombBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { ASSERT(TARRAY2_SIZE(writer->tombBlkArray) > 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index e87c00d382..b084ed13ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -97,7 +97,11 @@ int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row); int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData); int32_t tsdbDataFileFlush(SDataFileWriter *writer); +// tomb int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); +int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index eaa5e40726..10fa1e2b0b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -14,6 +14,7 @@ */ #include "tsdbSttFileRW.h" +#include "tsdbDataFileRW.h" // SSttFReader ============================================================ struct SSttFileReader { @@ -528,65 +529,6 @@ _exit: return code; } -int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range) { - int32_t code; - - if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; - - STombBlk tombBlk[1] = {{ - .dp[0] = - { - .offset = *fileSize, - .size = 0, - }, - .minTbid = - { - .suid = TARRAY2_FIRST(tombBlock->suid), - .uid = TARRAY2_FIRST(tombBlock->uid), - }, - .maxTbid = - { - .suid = TARRAY2_LAST(tombBlock->suid), - .uid = TARRAY2_LAST(tombBlock->uid), - }, - .minVer = TARRAY2_FIRST(tombBlock->version), - .maxVer = TARRAY2_FIRST(tombBlock->version), - .numRec = TOMB_BLOCK_SIZE(tombBlock), - .cmprAlg = cmprAlg, - }}; - - for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) { - if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) { - tombBlk->minVer = TARRAY2_GET(tombBlock->version, i); - } - if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) { - tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i); - } - } - - range->minVer = tombBlk->minVer; - range->maxVer = tombBlk->maxVer; - - for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { - code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), - TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); - if (code) return code; - - code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]); - if (code) return code; - - tombBlk->dp->size += tombBlk->size[i]; - *fileSize += tombBlk->size[i]; - } - - code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk); - if (code) return code; - - tTombBlockClear(tombBlock); - return 0; -} - static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; @@ -655,21 +597,6 @@ _exit: return code; } -int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { - ptr->size = TARRAY2_DATA_LEN(tombBlkArray); - if (ptr->size > 0) { - ptr->offset = *fileSize; - - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); - if (code) { - return code; - } - - *fileSize += ptr->size; - } - return 0; -} - static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 16f649cd9d..2a5b7f1080 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -20,6 +20,8 @@ extern void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t) extern int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); // new +#include "tsdbDataFileRW.h" +#include "tsdbSttFileRW.h" extern int32_t save_fs(const TFileSetArray *arr, const char *fname); extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); extern int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, @@ -30,9 +32,6 @@ extern int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); extern int32_t tsdbSttLvlClear(SSttLvl **lvl); extern int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); extern int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); -extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); -extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); extern int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { From 365d535e6d2a1df5f9a9a7e4b33b2f3f92132307 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 10:32:51 +0800 Subject: [PATCH 087/177] add rpc sync read timeout --- source/libs/transport/src/transCli.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3156a0733c..842f961141 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2576,12 +2576,14 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs STransMsg* pTransRsp = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); + taosMemoryFree(pTransRsp); return -1; } SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); if (pThrd == NULL) { transFreeMsg(pReq->pCont); + taosMemoryFree(pTransRsp); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return TSDB_CODE_RPC_BROKEN_LINK; } @@ -2644,12 +2646,14 @@ int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pRe STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); + taosMemoryFree(pTransMsg); return -1; } SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); if (pThrd == NULL) { transFreeMsg(pReq->pCont); + taosMemoryFree(pTransMsg); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return TSDB_CODE_RPC_BROKEN_LINK; } From 399afe1094d5ed1605270c36b8e6d81b70a43e75 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 15:45:19 +0800 Subject: [PATCH 088/177] refact: tidy up declarations of funcs in tsdbUpgrade.c --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 7 +++++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 3 +++ source/dnode/vnode/src/tsdb/tsdbFSet2.h | 3 +++ source/dnode/vnode/src/tsdb/tsdbSttFileRW.h | 3 +++ source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 18 +++++------------- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 83c14a50d0..4996c6c484 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -263,6 +263,7 @@ int32_t tsdbFSRollback(STsdb *pTsdb); int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS); void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS); +void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet); int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index b084ed13ae..a91852575d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -97,11 +97,18 @@ int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row); int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData); int32_t tsdbDataFileFlush(SDataFileWriter *writer); +// head +int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, + TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range); +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); + // tomb int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); +int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 8dfc86ee83..f8b87b8a84 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -72,6 +72,9 @@ int32_t tsdbFSEnableBgTask(STFileSystem *fs); // other int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); int32_t tsdbFSCheckCommit(STFileSystem *fs); +// utils +int32_t save_fs(const TFileSetArray *arr, const char *fname); +int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); struct STFSBgTask { EFSBgTaskT type; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 8155328e70..756250157b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -63,6 +63,9 @@ int64_t tsdbTFileSetMaxCid(const STFileSet *fset); SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level); // is empty bool tsdbTFileSetIsEmpty(const STFileSet *fset); +// stt +int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); +int32_t tsdbSttLvlClear(SSttLvl **lvl); struct STFileOp { tsdb_fop_t optype; diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h index d0481d5ec3..5b225da4e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h @@ -71,6 +71,9 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record); bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer); +int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); +int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); + struct SSttFileWriterConfig { STsdb *tsdb; int32_t maxRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 2a5b7f1080..24dfbf7450 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -16,23 +16,15 @@ #include "tsdbUpgrade.h" // old -extern void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); -extern int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); +#include "tsdb.h" +// extern void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); // new #include "tsdbDataFileRW.h" +#include "tsdbFS2.h" #include "tsdbSttFileRW.h" -extern int32_t save_fs(const TFileSetArray *arr, const char *fname); -extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); -extern int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, - TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range); -extern int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); -extern int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); -extern int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); -extern int32_t tsdbSttLvlClear(SSttLvl **lvl); -extern int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); -extern int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); -extern int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); +// extern int32_t save_fs(const TFileSetArray *arr, const char *fname); +// extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { int32_t code = 0; From f207b3ddd529cfc5c7bc2eb8b6402eb58775bd6b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 15:53:51 +0800 Subject: [PATCH 089/177] refact: rename func names of tsdbDataWriterUpdVerRange and tsdbSttWriterUpdVerRange --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 2798eb0291..6029ddcc29 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -642,7 +642,7 @@ _exit: return code; } -static int32_t tsdbSDataUpdVerRange(SDataFileWriterConfig *config, SVersionRange *range) { +static int32_t tsdbDataWriterUpdVerRange(SDataFileWriterConfig *config, SVersionRange *range) { config->minVer = TMIN(config->minVer, range->minVer); config->maxVer = TMAX(config->maxVer, range->maxVer); return 0; @@ -746,7 +746,7 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { &range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbSDataUpdVerRange(writer->config, &range); + tsdbDataWriterUpdVerRange(writer->config, &range); _exit: if (code) { @@ -813,7 +813,7 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData } SVersionRange range = {.minVer = record->minVer, .maxVer = record->maxVer}; - tsdbSDataUpdVerRange(writer->config, &range); + tsdbDataWriterUpdVerRange(writer->config, &range); // to .data file int32_t sizeArr[5] = {0}; @@ -1246,7 +1246,7 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbSDataUpdVerRange(writer->config, &range); + tsdbDataWriterUpdVerRange(writer->config, &range); _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 10fa1e2b0b..1b1438168d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -402,7 +402,7 @@ struct SSttFileWriter { uint8_t *bufArr[5]; }; -static int32_t tsdbSSttUpdVerRange(SSttFileWriterConfig *config, SVersionRange *range) { +static int32_t tsdbSttWriterUpdVerRange(SSttFileWriterConfig *config, SVersionRange *range) { config->minVer = TMIN(config->minVer, range->minVer); config->maxVer = TMAX(config->maxVer, range->maxVer); return 0; @@ -469,7 +469,7 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, writer->sttBlkArray, writer->config->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbSSttUpdVerRange(writer->config, &range); + tsdbSttWriterUpdVerRange(writer->config, &range); _exit: if (code) { @@ -539,7 +539,7 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, writer->tombBlkArray, writer->config->bufArr, &range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbSSttUpdVerRange(writer->config, &range); + tsdbSttWriterUpdVerRange(writer->config, &range); _exit: if (code) { From 01d0c1247d2de0500a4366e712ac7e9ad9564756 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 19:13:58 +0800 Subject: [PATCH 090/177] enh: record ver range in writer ctx --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 61 +++++++++----------- source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 5 +- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 4 -- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 32 ++++------ source/dnode/vnode/src/tsdb/tsdbSttFileRW.h | 2 - 5 files changed, 42 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 6029ddcc29..08771efb04 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -487,6 +487,8 @@ struct SDataFileWriter { int32_t tombBlkArrayIdx; STombBlock tombBlock[1]; int32_t tombBlockIdx; + // range + SVersionRange range; } ctx[1]; STFile files[TSDB_FTYPE_MAX]; @@ -633,6 +635,9 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .maxVer = VERSION_MIN, }; + // range + writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + writer->ctx->opened = true; _exit: @@ -642,9 +647,9 @@ _exit: return code; } -static int32_t tsdbDataWriterUpdVerRange(SDataFileWriterConfig *config, SVersionRange *range) { - config->minVer = TMIN(config->minVer, range->minVer); - config->maxVer = TMAX(config->maxVer, range->maxVer); +int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer) { + range->minVer = TMIN(range->minVer, minVer); + range->maxVer = TMAX(range->maxVer, maxVer); return 0; } @@ -688,8 +693,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl } } - range->minVer = brinBlk->minVer; - range->maxVer = brinBlk->maxVer; + tsdbWriterUpdVerRange(range, brinBlk->minVer, brinBlk->maxVer); // write to file for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { @@ -739,15 +743,12 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr, - &range); + &writer->ctx->range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbDataWriterUpdVerRange(writer->config, &range); - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); @@ -812,8 +813,7 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData } } - SVersionRange range = {.minVer = record->minVer, .maxVer = record->maxVer}; - tsdbDataWriterUpdVerRange(writer->config, &range); + tsdbWriterUpdVerRange(&writer->ctx->range, record->minVer, record->maxVer); // to .data file int32_t sizeArr[5] = {0}; @@ -1200,8 +1200,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl } } - range->minVer = tombBlk->minVer; - range->maxVer = tombBlk->maxVer; + tsdbWriterUpdVerRange(range, tombBlk->minVer, tombBlk->maxVer); for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), @@ -1241,12 +1240,11 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - - SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, - &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr, &range); + &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr, + &writer->ctx->range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbDataWriterUpdVerRange(writer->config, &range); _exit: if (code) { @@ -1402,6 +1400,12 @@ _exit: return code; } +int32_t tsdbTFileUpdVerRange(STFile *f, SVersionRange range) { + f->minVer = TMIN(f->minVer, range.minVer); + f->maxVer = TMAX(f->maxVer, range.maxVer); + return 0; +} + static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArray *opArr) { int32_t code = 0; int32_t lino = 0; @@ -1446,8 +1450,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); @@ -1459,8 +1462,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { @@ -1470,8 +1472,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .of = writer->config->files[ftype].file, .nf = writer->files[ftype], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1484,8 +1485,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { @@ -1495,8 +1495,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .of = writer->config->files[ftype].file, .nf = writer->files[ftype], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1536,8 +1535,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1547,14 +1545,9 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr code = tsdbFsyncFile(writer->fd[i]); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&writer->fd[i]); - writer->files[i].minVer = TMIN(writer->files[i].minVer, writer->config->minVer); - writer->files[i].maxVer = TMAX(writer->files[i].maxVer, writer->config->maxVer); } } - writer->config->minVer = VERSION_MAX; - writer->config->maxVer = VERSION_MIN; - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index a91852575d..ca55a5420a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -76,8 +76,6 @@ typedef struct SDataFileWriterConfig { int32_t maxRow; int32_t szPage; int32_t fid; - int64_t minVer; - int64_t maxVer; int64_t cid; SDiskID did; int64_t compactVersion; @@ -110,6 +108,9 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); +// utils +int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index 4397d1cb5c..e6b3cf8f54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -143,8 +143,6 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .maxRow = config->maxRow, .szPage = config->szPage, .fid = config->fid, - .minVer = VERSION_MAX, - .maxVer = VERSION_MIN, .cid = config->cid, .did = config->did, .compactVersion = config->compactVersion, @@ -170,8 +168,6 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .compactVersion = config->compactVersion, .did = config->did, .fid = config->fid, - .minVer = VERSION_MAX, - .maxVer = VERSION_MIN, .cid = config->cid, .level = config->level, .skmTb = writer[0]->skmTb, diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 1b1438168d..c1bf8bb027 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -384,6 +384,8 @@ struct SSttFileWriter { struct { bool opened; TABLEID tbid[1]; + // range + SVersionRange range; } ctx[1]; // file STsdbFD *fd; @@ -402,12 +404,6 @@ struct SSttFileWriter { uint8_t *bufArr[5]; }; -static int32_t tsdbSttWriterUpdVerRange(SSttFileWriterConfig *config, SVersionRange *range) { - config->minVer = TMIN(config->minVer, range->minVer); - config->maxVer = TMAX(config->maxVer, range->maxVer); - return 0; -} - static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) { if (blockData->nRow == 0) return 0; @@ -432,8 +428,7 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, i if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow]; } - range->minVer = sttBlk->minVer; - range->maxVer = sttBlk->maxVer; + tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer); int32_t sizeArr[5] = {0}; code = tCmprBlockData(blockData, cmprAlg, NULL, NULL, bufArr, sizeArr); @@ -465,11 +460,9 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, - writer->sttBlkArray, writer->config->bufArr, &range); + writer->sttBlkArray, writer->config->bufArr, &writer->ctx->range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbSttWriterUpdVerRange(writer->config, &range); _exit: if (code) { @@ -535,11 +528,9 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, - writer->tombBlkArray, writer->config->bufArr, &range); + writer->tombBlkArray, writer->config->bufArr, &writer->ctx->range); TSDB_CHECK_CODE(code, lino, _exit); - tsdbSttWriterUpdVerRange(writer->config, &range); _exit: if (code) { @@ -637,8 +628,8 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, - .minVer = writer->config->minVer, - .maxVer = writer->config->maxVer, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, .stt[0] = { .level = writer->config->level, @@ -658,6 +649,9 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { TSDB_CHECK_CODE(code, lino, _exit); writer->file->size += sizeof(hdr); + // range + writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + writer->ctx->opened = true; _exit: @@ -727,10 +721,8 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o .fid = writer->config->fid, .nf = writer->file[0], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->config->minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->config->maxVer); - writer->config->minVer = VERSION_MAX; - writer->config->maxVer = VERSION_MIN; + op.nf.minVer = TMIN(op.nf.minVer, writer->ctx->range.minVer); + op.nf.maxVer = TMAX(op.nf.maxVer, writer->ctx->range.maxVer); code = TARRAY2_APPEND(opArray, op); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h index 5b225da4e3..0051a6cd92 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h @@ -82,8 +82,6 @@ struct SSttFileWriterConfig { int64_t compactVersion; SDiskID did; int32_t fid; - int64_t minVer; - int64_t maxVer; int64_t cid; int32_t level; SSkmInfo *skmTb; From f8c6e8744a5595c84028667e484e820d0485e68a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 31 Aug 2023 19:39:47 +0800 Subject: [PATCH 091/177] refact: change field name of pExclude in SVSnapReader --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 5 +++-- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index cc355c5f32..902cc782ab 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -295,7 +295,7 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback); // STsdbSnapReader ======================================== -int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pEx, +int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pExclude, STsdbSnapReader** ppReader); int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader); int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 0a8bb4c37a..0c9c47fb98 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -991,7 +991,7 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pEx, +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pExclude, TSnapRangeArray **fsrArr) { int32_t code = 0; STFileSet *fset; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index b604256e89..2fdad2d662 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -412,7 +412,8 @@ _exit: return code; } -int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, void* pEx, STsdbSnapReader** reader) { +int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, void* pExclude, + STsdbSnapReader** reader) { int32_t code = 0; int32_t lino = 0; @@ -424,7 +425,7 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pEx, &reader[0]->fsrArr); + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pExclude, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index b89fed73f8..6beb8e8dbf 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -29,7 +29,7 @@ struct SVSnapReader { SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; - TSnapRangeArray *pEx; + TSnapRangeArray *pExclude; STsdbSnapReader *pTsdbReader; // tq int8_t tqHandleDone; @@ -179,7 +179,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { - code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pEx, + code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pExclude, &pReader->pTsdbReader); if (code) goto _err; } From dc402e21e36c05427e6be245827d218ec3670ffd Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Sep 2023 17:32:59 +0800 Subject: [PATCH 092/177] fix: merge old version range of head and tomb files in tsdbDataFileWriterCloseCommit --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 08771efb04..e6f1d29b22 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -1434,6 +1434,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr code = tsdbDataFileWriteHeadFooter(writer); TSDB_CHECK_CODE(code, lino, _exit); + SVersionRange ofRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + // .head ftype = TSDB_FTYPE_HEAD; if (writer->config->files[ftype].exist) { @@ -1442,6 +1444,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .of = writer->config->files[ftype].file, }; + ofRange = (SVersionRange){.minVer = op.of.minVer, .maxVer = op.of.maxVer}; code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1450,6 +1453,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; + tsdbTFileUpdVerRange(&op.nf, ofRange); tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); @@ -1520,6 +1524,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr code = tsdbDataFileWriteTombFooter(writer); TSDB_CHECK_CODE(code, lino, _exit); + SVersionRange ofRange = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + ftype = TSDB_FTYPE_TOMB; if (writer->config->files[ftype].exist) { op = (STFileOp){ @@ -1527,6 +1533,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .of = writer->config->files[ftype].file, }; + ofRange = (SVersionRange){.minVer = op.of.minVer, .maxVer = op.of.maxVer}; code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1535,6 +1542,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .fid = writer->config->fid, .nf = writer->files[ftype], }; + tsdbTFileUpdVerRange(&op.nf, ofRange); tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); From 6b753cc0cdda5547337b6c778b810fa58673823e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Sep 2023 17:39:55 +0800 Subject: [PATCH 093/177] refact: remove unused code is_same_file --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 0c9c47fb98..d6c81b1fe5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -258,14 +258,6 @@ _exit: return code; } -static bool is_same_file(const STFile *f1, const STFile f2) { - if (f1->type != f2.type) return false; - if (f1->did.level != f2.did.level) return false; - if (f1->did.id != f2.did.id) return false; - if (f1->cid != f2.cid) return false; - return true; -} - static int32_t apply_commit(STFileSystem *fs) { int32_t code = 0; TFileSetArray *fsetArray1 = fs->fSetArr; From 54a10154a4459af056b3f63b0d4557d40f14c472 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Sep 2023 18:12:45 +0800 Subject: [PATCH 094/177] fixup: remove unused minVer and maxVer in ctx of SCommitter2 --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 302991fb0f..79964c5636 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -43,8 +43,6 @@ typedef struct { SDiskID did; TSKEY minKey; TSKEY maxKey; - int64_t minVer; - int64_t maxVer; STFileSet *fset; TABLEID tbid[1]; bool hasTSData; From 0a67cc84778b1fd07a1345d85c43315f15ab14e2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Sep 2023 19:06:48 +0800 Subject: [PATCH 095/177] refact: use tsdbTFileUpdVerRange in tsdbSttFWriterCloseCommit --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 1 + source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index ca55a5420a..c4aed6e787 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -110,6 +110,7 @@ int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t // utils int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); +int32_t tsdbTFileUpdVerRange(STFile *f, SVersionRange range); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index c1bf8bb027..7c3b185e20 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -721,8 +721,7 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o .fid = writer->config->fid, .nf = writer->file[0], }; - op.nf.minVer = TMIN(op.nf.minVer, writer->ctx->range.minVer); - op.nf.maxVer = TMAX(op.nf.maxVer, writer->ctx->range.maxVer); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); code = TARRAY2_APPEND(opArray, op); TSDB_CHECK_CODE(code, lino, _exit); From 6ae9bd0e9a09a1115396c882e648d45d1b5f2240 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 5 Sep 2023 10:20:37 +0800 Subject: [PATCH 096/177] feat: add tsdbFSToSnapRangeArray --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index d6c81b1fe5..fed43a16c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -983,6 +983,81 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } +int32_t tsdbTSnapRangeCmprFn(STSnapRange *fsr1, STSnapRange *fsr2) { + if (fsr1->fid < fsr2->fid) return -1; + if (fsr1->fid > fsr2->fid) return 1; + if (fsr1->sver < fsr2->sver) return -1; + if (fsr1->sver > fsr2->sver) return 1; + if (fsr1->ever < fsr2->ever) return -1; + if (fsr1->ever < fsr2->ever) return 1; + return 0; +} + +int32_t tsdbTFileInsertSnapRange(STFile *f, TSnapRangeArray *snapR) { + STSnapRange *fsr = taosMemoryCalloc(1, sizeof(*fsr)); + if (fsr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + fsr->fid = f->fid; + fsr->sver = f->minVer; + fsr->ever = f->maxVer; + + int32_t code = TARRAY2_SORT_INSERT(snapR, fsr, tsdbTSnapRangeCmprFn); + if (code) { + taosMemoryFree(fsr); + fsr = NULL; + } + return code; +} + +int32_t tsdbTFSetInsertSnapRange(STFileSet *fset, TSnapRangeArray *snapR) { + STFile tf = {.fid = fset->fid, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + STFile *f = fset->farr[ftype]->f; + tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = f->minVer, .maxVer = f->maxVer}); + } + + int32_t code = tsdbTFileInsertSnapRange(&tf, snapR); + if (code) return code; + + const SSttLvl *lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + STFileObj *fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + code = tsdbTFileInsertSnapRange(fobj->f, snapR); + if (code) return code; + } + } + return code; +} + +TSnapRangeArray *tsdbFSToSnapRangeArray(STFileSystem *fs) { + int32_t code = 0; + TSnapRangeArray *snapR = taosMemoryCalloc(1, sizeof(*snapR)); + if (snapR == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + TARRAY2_INIT(snapR); + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + STFileSet *fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTFSetInsertSnapRange(fset, snapR); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(snapR, tsdbTSnapRangeClear); + taosMemoryFree(snapR); + snapR = NULL; + } + return snapR; +} + int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pExclude, TSnapRangeArray **fsrArr) { int32_t code = 0; From 44a7f5df4799cf9802cfd2bb1421691229964627 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 5 Sep 2023 15:14:43 +0800 Subject: [PATCH 097/177] feat: add tsdbSnapDiff --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 254 ++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index fed43a16c1..231cd95658 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -983,7 +983,7 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbTSnapRangeCmprFn(STSnapRange *fsr1, STSnapRange *fsr2) { +static int32_t tsdbTSnapRangeCmprFn(STSnapRange *fsr1, STSnapRange *fsr2) { if (fsr1->fid < fsr2->fid) return -1; if (fsr1->fid > fsr2->fid) return 1; if (fsr1->sver < fsr2->sver) return -1; @@ -993,7 +993,7 @@ int32_t tsdbTSnapRangeCmprFn(STSnapRange *fsr1, STSnapRange *fsr2) { return 0; } -int32_t tsdbTFileInsertSnapRange(STFile *f, TSnapRangeArray *snapR) { +static int32_t tsdbTFileInsertSnapRange(STFile *f, TSnapRangeArray *snapR) { STSnapRange *fsr = taosMemoryCalloc(1, sizeof(*fsr)); if (fsr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1011,7 +1011,7 @@ int32_t tsdbTFileInsertSnapRange(STFile *f, TSnapRangeArray *snapR) { return code; } -int32_t tsdbTFSetInsertSnapRange(STFileSet *fset, TSnapRangeArray *snapR) { +static int32_t tsdbTFSetInsertSnapRange(STFileSet *fset, TSnapRangeArray *snapR) { STFile tf = {.fid = fset->fid, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset->farr[ftype] == NULL) continue; @@ -1058,24 +1058,246 @@ TSnapRangeArray *tsdbFSToSnapRangeArray(STFileSystem *fs) { return snapR; } +static STSnapRange *taosDupSnapRange(STSnapRange *x) { + STSnapRange *y = taosMemoryCalloc(1, sizeof(STSnapRange)); + if (y == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + ASSERTS(terrno == 0, "Out of memory"); + return NULL; + } + y->fid = x->fid; + y->sver = x->sver; + y->ever = x->ever; + return y; +} + +static TSnapRangeArray *taosDupSnapRangeArray(const TSnapRangeArray *X) { + TSnapRangeArray *Y = taosMemoryCalloc(1, sizeof(*Y)); + if (Y == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + TARRAY2_INIT(Y); + + if (X) { + STSnapRange *x; + TARRAY2_FOREACH(X, x) { + STSnapRange *tp = taosDupSnapRange(x); + TARRAY2_APPEND(Y, tp); + } + } + return Y; +} + +static TSnapRangeArray *tsdbSnapDiff(const TSnapRangeArray *snapR, const TSnapRangeArray *pExclude) { + TSnapRangeArray *Z = NULL; + TSnapRangeArray *U = NULL; + TSnapRangeArray *V = NULL; + TSnapRangeArray *X = taosDupSnapRangeArray(snapR); + TSnapRangeArray *Y = taosDupSnapRangeArray(pExclude); + int32_t code = -1; + + // separate intersections of snap ranges + U = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); + if (U == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + TARRAY2_INIT(U); + + V = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); + if (V == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + TARRAY2_INIT(V); + + int32_t i = 0; + int32_t j = 0; + while (i < TARRAY2_SIZE(X) && j < TARRAY2_SIZE(Y)) { + STSnapRange *x = TARRAY2_GET(X, i); + STSnapRange *y = TARRAY2_GET(Y, j); + + if (x->fid < y->fid) { + STSnapRange *tmp = taosDupSnapRange(x); + TARRAY2_APPEND(U, tmp); + i++; + } else if (x->fid > y->fid) { + STSnapRange *tmp = taosDupSnapRange(y); + TARRAY2_APPEND(V, tmp); + j++; + } else { + if (x->sver < y->sver) { + if (x->ever < y->ever) { + STSnapRange *tmp = taosDupSnapRange(x); + TARRAY2_APPEND(U, tmp); + i++; + } else { + STSnapRange *tmp = taosDupSnapRange(x); + tmp->ever = y->sver - 1; + TARRAY2_APPEND(U, tmp); + x->sver = y->sver; + } + } else if (x->sver > y->sver) { + if (y->ever < x->ever) { + STSnapRange *tmp = taosDupSnapRange(y); + TARRAY2_APPEND(V, tmp); + j++; + } else { + STSnapRange *tmp = taosDupSnapRange(y); + tmp->ever = x->sver - 1; + TARRAY2_APPEND(V, tmp); + y->sver = x->sver; + } + } else { + if (x->ever < y->ever) { + STSnapRange *tmp = taosDupSnapRange(x); + TARRAY2_APPEND(U, tmp); + i++; + tmp = taosDupSnapRange(y); + tmp->ever = x->ever; + TARRAY2_APPEND(V, tmp); + y->sver = x->ever + 1; + } else if (x->ever > y->ever) { + STSnapRange *tmp = taosDupSnapRange(y); + TARRAY2_APPEND(V, tmp); + j++; + tmp = taosDupSnapRange(x); + tmp->ever = y->ever; + TARRAY2_APPEND(U, tmp); + x->sver = y->ever + 1; + } else { + STSnapRange *tmp = taosDupSnapRange(x); + TARRAY2_APPEND(U, tmp); + i++; + tmp = taosDupSnapRange(y); + TARRAY2_APPEND(V, tmp); + j++; + } + } + } + } + while (i < TARRAY2_SIZE(X)) { + STSnapRange *x = TARRAY2_GET(X, i); + STSnapRange *tmp = taosDupSnapRange(x); + TARRAY2_APPEND(U, tmp); + i++; + } + while (j < TARRAY2_SIZE(Y)) { + STSnapRange *y = TARRAY2_GET(Y, j); + STSnapRange *tmp = taosDupSnapRange(y); + TARRAY2_APPEND(V, tmp); + j++; + } + + // difference of snap ranges + Z = taosMemoryCalloc(1, sizeof(*Z)); + if (Z == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + TARRAY2_INIT(Z); + + i = 0; + j = 0; + while (i < TARRAY2_SIZE(U) && j < TARRAY2_SIZE(V)) { + STSnapRange *u = TARRAY2_GET(U, i); + STSnapRange *v = TARRAY2_GET(V, j); + + if (u->fid < v->fid) { + STSnapRange *tmp = taosDupSnapRange(u); + TARRAY2_APPEND(Z, tmp); + i++; + } else if (u->fid == v->fid) { + if (u->sver < v->sver) { + STSnapRange *tmp = taosDupSnapRange(u); + TARRAY2_APPEND(Z, tmp); + i++; + } else if (u->sver > v->sver) { + ASSERT(u->ever > v->ever); + j++; + } else { + ASSERT(u->ever == v->ever); + i++; + j++; + } + } + } + while (i < TARRAY2_SIZE(U)) { + STSnapRange *u = TARRAY2_GET(U, i); + STSnapRange *tmp = taosDupSnapRange(u); + TARRAY2_APPEND(Z, tmp); + i++; + } + + code = 0; +_out: + TSnapRangeArray **ppArrs[4] = {&X, &Y, &U, &V}; + int len = sizeof(ppArrs) / sizeof(ppArrs[0]); + for (int i = 0; i < len; i++) { + if (ppArrs[i][0] == NULL) continue; + TARRAY2_DESTROY(ppArrs[i][0], tsdbTSnapRangeClear); + taosMemoryFree(ppArrs[i][0]); + ppArrs[i][0] = NULL; + } + if (code != 0 && Z) { + TARRAY2_DESTROY(Z, tsdbTSnapRangeClear); + taosMemoryFree(Z); + Z = NULL; + } + return Z; +} + int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pExclude, TSnapRangeArray **fsrArr) { - int32_t code = 0; + int32_t code = -1; STFileSet *fset; STSnapRange *fsr1 = NULL; - fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); - if (fsrArr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + TSnapRangeArray *snapF = tsdbFSToSnapRangeArray(fs); + if (snapF == NULL) { + tsdbError("failed to generate snap ranges from fs since %s.", terrstr()); + goto _out; + } + TSnapRangeArray *snapD = tsdbSnapDiff(snapF, pExclude); + if (snapD == NULL) { + tsdbError("failed to get diff of snap ranges since %s.", terrstr()); + goto _out; + } + fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); + if (fsrArr[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + + int32_t i = 0; taosThreadRwlockRdlock(&fs->tsdb->rwLock); TARRAY2_FOREACH(fs->fSetArr, fset) { - code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver, ever, &fsr1); - if (code) break; + while (i < TARRAY2_SIZE(snapD)) { + STSnapRange *u = TARRAY2_GET(snapD, i); + if (fset->fid < u->fid) { + break; + } else if (fset->fid > u->fid) { + i++; + continue; + } else { + i++; + } + int64_t sver1 = TMAX(sver, u->sver); + int64_t ever1 = TMIN(ever, u->ever); + if (sver1 > ever1) { + continue; + } + code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); + if (code) break; - code = TARRAY2_APPEND(fsrArr[0], fsr1); - if (code) break; + code = TARRAY2_APPEND(fsrArr[0], fsr1); + if (code) break; - fsr1 = NULL; + fsr1 = NULL; + } + if (code) break; } taosThreadRwlockUnlock(&fs->tsdb->rwLock); @@ -1084,6 +1306,16 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); fsrArr[0] = NULL; } + +_out: + TSnapRangeArray **ppArrs[2] = {&snapF, &snapD}; + int len = sizeof(ppArrs) / sizeof(ppArrs[0]); + for (int i = 0; i < len; i++) { + if (ppArrs[i][0] == NULL) continue; + TARRAY2_DESTROY(ppArrs[i][0], tsdbTSnapRangeClear); + taosMemoryFree(ppArrs[i][0]); + ppArrs[i][0] = NULL; + } return code; } From 4540bcb170d209c8f755ab7b5f95061fb96e85de Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 5 Sep 2023 15:33:35 +0800 Subject: [PATCH 098/177] fixup: work-around a compiler bug --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 231cd95658..78480988cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1234,11 +1234,11 @@ static TSnapRangeArray *tsdbSnapDiff(const TSnapRangeArray *snapR, const TSnapRa _out: TSnapRangeArray **ppArrs[4] = {&X, &Y, &U, &V}; int len = sizeof(ppArrs) / sizeof(ppArrs[0]); - for (int i = 0; i < len; i++) { - if (ppArrs[i][0] == NULL) continue; - TARRAY2_DESTROY(ppArrs[i][0], tsdbTSnapRangeClear); - taosMemoryFree(ppArrs[i][0]); - ppArrs[i][0] = NULL; + for (int k = 0; k < len; k++) { + if (ppArrs[k][0] == NULL) continue; + TARRAY2_DESTROY(ppArrs[k][0], tsdbTSnapRangeClear); + taosMemoryFree(ppArrs[k][0]); + ppArrs[k][0] = NULL; } if (code != 0 && Z) { TARRAY2_DESTROY(Z, tsdbTSnapRangeClear); @@ -1310,11 +1310,11 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev _out: TSnapRangeArray **ppArrs[2] = {&snapF, &snapD}; int len = sizeof(ppArrs) / sizeof(ppArrs[0]); - for (int i = 0; i < len; i++) { - if (ppArrs[i][0] == NULL) continue; - TARRAY2_DESTROY(ppArrs[i][0], tsdbTSnapRangeClear); - taosMemoryFree(ppArrs[i][0]); - ppArrs[i][0] = NULL; + for (int k = 0; k < len; k++) { + if (ppArrs[k][0] == NULL) continue; + TARRAY2_DESTROY(ppArrs[k][0], tsdbTSnapRangeClear); + taosMemoryFree(ppArrs[k][0]); + ppArrs[k][0] = NULL; } return code; } From 0ddbcd50d0e5cd6f17954f9313bce283c41a582c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Sep 2023 11:43:13 +0800 Subject: [PATCH 099/177] enh: maintain independent version range of tombs for SDataFileWriter --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 8 +++++--- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index e6f1d29b22..df6b85a889 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -488,7 +488,8 @@ struct SDataFileWriter { STombBlock tombBlock[1]; int32_t tombBlockIdx; // range - SVersionRange range; + SVersionRange range; + SVersionRange tombRange; } ctx[1]; STFile files[TSDB_FTYPE_MAX]; @@ -637,6 +638,7 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { // range writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + writer->ctx->tombRange = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; writer->ctx->opened = true; @@ -1243,7 +1245,7 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr, - &writer->ctx->range); + &writer->ctx->tombRange); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1543,7 +1545,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .nf = writer->files[ftype], }; tsdbTFileUpdVerRange(&op.nf, ofRange); - tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); + tsdbTFileUpdVerRange(&op.nf, writer->ctx->tombRange); code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 24dfbf7450..876c0df4a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -486,9 +486,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TSDB_CHECK_CODE(code, lino, _exit); } - SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->bufArr, &range); + ctx->bufArr, &tombRange); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -499,9 +499,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TSDB_CHECK_CODE(code, lino, _exit); } - SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->bufArr, &range); + ctx->bufArr, &tombRange); TSDB_CHECK_CODE(code, lino, _exit); } From b23bcee690b0b4d5f8aed86d9a6a5828b13381ae Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 11 Sep 2023 10:54:28 +0800 Subject: [PATCH 100/177] fixup: about to revert this commit --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 78480988cf..b082c450f5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1026,10 +1026,14 @@ static int32_t tsdbTFSetInsertSnapRange(STFileSet *fset, TSnapRangeArray *snapR) TARRAY2_FOREACH(fset->lvlArr, lvl) { STFileObj *fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { + // tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = fobj->f->minVer, .maxVer = fobj->f->maxVer}); code = tsdbTFileInsertSnapRange(fobj->f, snapR); if (code) return code; } } + + // int32_t code = tsdbTFileInsertSnapRange(&tf, snapR); + // if (code) return code; return code; } @@ -1272,6 +1276,10 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev } int32_t i = 0; + code = 0; + + // TODO: use the same fs fSetArr as get snapDiff. The following treatment is potentially wrong + // if the fSetArr are changed. taosThreadRwlockRdlock(&fs->tsdb->rwLock); TARRAY2_FOREACH(fs->fSetArr, fset) { while (i < TARRAY2_SIZE(snapD)) { @@ -1279,6 +1287,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev if (fset->fid < u->fid) { break; } else if (fset->fid > u->fid) { + ASSERT(false); i++; continue; } else { @@ -1289,6 +1298,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev if (sver1 > ever1) { continue; } + tsdbInfo("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); + code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); if (code) break; From 083dd148be6e7fe9569fbb9ae82f3d24375b1ef6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 11 Sep 2023 19:05:40 +0800 Subject: [PATCH 101/177] feat: exchange difference of snapshot info for replication --- include/libs/sync/sync.h | 12 +++- source/dnode/mnode/impl/src/mndSync.c | 3 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 8 ++- source/dnode/vnode/src/vnd/vnodeSync.c | 5 +- source/libs/sync/inc/syncMessage.h | 7 ++- source/libs/sync/src/syncMessage.c | 4 +- source/libs/sync/src/syncSnapshot.c | 82 ++++++++++++++++++++++---- 8 files changed, 100 insertions(+), 23 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index f69afbd71b..53e6ec0d71 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -87,6 +87,12 @@ typedef enum { TAOS_SYNC_ROLE_ERROR = 2, } ESyncRole; +typedef enum { + TAOS_SYNC_SNAP_INFO_BRIEF = 0, + TAOS_SYNC_SNAP_INFO_FULL = 1, + TAOS_SYNC_SNAP_INFO_DIFF = 2, +} ESyncSnapInfoTyp; + typedef struct SNodeInfo { int64_t clusterId; int32_t nodeId; @@ -139,10 +145,12 @@ typedef struct SReConfigCbMeta { typedef struct SSnapshotParam { SyncIndex start; SyncIndex end; + void* data; // with SMsgHead } SSnapshotParam; typedef struct SSnapshot { - void* data; + ESyncSnapInfoTyp typ; + void* data; // with SMsgHead SyncIndex lastApplyIndex; SyncTerm lastApplyTerm; SyncIndex lastConfigIndex; @@ -171,7 +179,7 @@ typedef struct SSyncFSM { void (*FpBecomeLearnerCb)(const struct SSyncFSM* pFsm); int32_t (*FpGetSnapshot)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); - void (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot); + int32_t (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot); int32_t (*FpSnapshotStartRead)(const struct SSyncFSM* pFsm, void* pReaderParam, void** ppReader); void (*FpSnapshotStopRead)(const struct SSyncFSM* pFsm, void* pReader); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 5759737a6a..7f6a0397ad 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -286,9 +286,10 @@ int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pRe return 0; } -static void mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) { +static int32_t mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) { SMnode *pMnode = pFsm->data; sdbGetCommitInfo(pMnode->pSdb, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, &pSnapshot->lastConfigIndex); + return 0; } void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c40e2657f9..ba7bad67e4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -69,7 +69,7 @@ int32_t vnodeBegin(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); -void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); +int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index db94f32459..9228269992 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -518,9 +518,13 @@ void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } -void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { - pSnapshot->data = NULL; +int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; pSnapshot->lastApplyTerm = pVnode->state.commitTerm; pSnapshot->lastConfigIndex = -1; + + if (pSnapshot->typ == TAOS_SYNC_SNAP_INFO_FULL) { + // TODO: get full info of snapshots + } + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index a6c743c87d..b9f2d23c7b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -416,8 +416,8 @@ static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } -static void vnodeSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) { - vnodeGetSnapshot(pFsm->data, pSnapshot); +static int32_t vnodeSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) { + return vnodeGetSnapshot(pFsm->data, pSnapshot); } static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { @@ -642,6 +642,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex; pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg; pFsm->FpRollBackCb = vnodeSyncRollBackMsg; + pFsm->FpGetSnapshot = NULL; pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshotInfo; pFsm->FpRestoreFinishCb = vnodeRestoreFinish; pFsm->FpLeaderTransferCb = NULL; diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index f8c96d8be2..c0d3663a8f 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -200,7 +200,7 @@ typedef struct SyncSnapshotSend { SSyncCfg lastConfig; int64_t startTime; int32_t seq; - int16_t reserved; + int16_t payloadType; uint32_t dataLen; char data[]; } SyncSnapshotSend; @@ -219,7 +219,8 @@ typedef struct SyncSnapshotRsp { int32_t ack; int32_t code; SyncIndex snapBeginIndex; // when ack = SYNC_SNAPSHOT_SEQ_BEGIN, it's valid - int16_t reserved; + int16_t payloadType; + char data[]; } SyncSnapshotRsp; typedef struct SyncLeaderTransfer { @@ -267,7 +268,7 @@ int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildPreSnapshotReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildApplyMsg(SRpcMsg* pMsg, const SRpcMsg* pOriginal, int32_t vgId, SFsmCbMeta* pMeta); int32_t syncBuildSnapshotSend(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId); -int32_t syncBuildSnapshotSendRsp(SRpcMsg* pMsg, int32_t vgId); +int32_t syncBuildSnapshotSendRsp(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId); int32_t syncBuildLeaderTransfer(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildLocalCmd(SRpcMsg* pMsg, int32_t vgId); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 72c8887803..00ca6d8f90 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -270,8 +270,8 @@ int32_t syncBuildSnapshotSend(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId) { return 0; } -int32_t syncBuildSnapshotSendRsp(SRpcMsg* pMsg, int32_t vgId) { - int32_t bytes = sizeof(SyncSnapshotRsp); +int32_t syncBuildSnapshotSendRsp(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId) { + int32_t bytes = sizeof(SyncSnapshotRsp) + dataLen; pMsg->pCont = rpcMallocCont(bytes); pMsg->msgType = TDMT_SYNC_SNAPSHOT_RSP; pMsg->contLen = bytes; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 763d4ec5d6..00dcd7e949 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -74,6 +74,7 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { + int32_t code = -1; pSender->start = true; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; @@ -95,11 +96,26 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->lastSendTime = pSender->startTime; pSender->finish = false; + // Get full snapshot info + SSyncNode *pSyncNode = pSender->pSyncNode; + SSnapshot snapInfo = {.typ = TAOS_SYNC_SNAP_INFO_FULL}; + if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) { + sSError(pSender, "snapshot get info failure since %s", terrstr()); + goto _out; + } + + int dataLen = 0; + if (snapInfo.data) { + SMsgHead *msgHead = snapInfo.data; + ASSERT(msgHead->vgId == pSyncNode->vgId); + dataLen = sizeof(SMsgHead) + msgHead->contLen; + } + // build begin msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSend(&rpcMsg, 0, pSender->pSyncNode->vgId) != 0) { + if (syncBuildSnapshotSend(&rpcMsg, dataLen, pSender->pSyncNode->vgId) != 0) { sSError(pSender, "snapshot sender build msg failed since %s", terrstr()); - return -1; + goto _out; } SyncSnapshotSend *pMsg = rpcMsg.pCont; @@ -114,16 +130,27 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->startTime = pSender->startTime; pMsg->seq = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; + if (dataLen > 0) { + pMsg->payloadType = snapInfo.typ; + memcpy(pMsg->data, snapInfo.data, dataLen); + } + // event log syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender start"); // send msg if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { sSError(pSender, "snapshot sender send msg failed since %s", terrstr()); - return -1; + goto _out; } - return 0; + code = 0; +_out: + if (snapInfo.data) { + taosMemoryFree(snapInfo.data); + snapInfo.data = NULL; + } + return code; } void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { @@ -578,10 +605,29 @@ _SEND_REPLY: // build msg ; // make complier happy + code = -1; + SSnapshot snapInfo = {.typ = TAOS_SYNC_SNAP_INFO_DIFF}; + int32_t dataLen = 0; + if (pMsg->dataLen > 0) { + void *data = taosMemoryCalloc(1, pMsg->dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + memcpy(data, pMsg->data, dataLen); + snapInfo.data = data; + data = NULL; + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); + + SMsgHead *msgHead = snapInfo.data; + ASSERT(msgHead->vgId == pSyncNode->vgId); + dataLen = msgHead->contLen; + } + SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, pSyncNode->vgId) != 0) { + if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) { sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); - return -1; + goto _out; } SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; @@ -595,13 +641,24 @@ _SEND_REPLY: pRspMsg->code = code; pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); + if (snapInfo.data) { + pRspMsg->payloadType = snapInfo.typ; + memcpy(pRspMsg->data, snapInfo.data, dataLen); + } + // send msg syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver pre-snapshot"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); - return -1; + goto _out; } + code = 0; +_out: + if (snapInfo.data) { + taosMemoryFree(snapInfo.data); + snapInfo.data = NULL; + } return code; } @@ -635,7 +692,7 @@ _SEND_REPLY: // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, pSyncNode->vgId) != 0) { + if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { sRError(pReceiver, "snapshot receiver build resp failed since %s", terrstr()); return -1; } @@ -685,7 +742,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, pSyncNode->vgId)) { + if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { sRError(pReceiver, "snapshot receiver build resp failed since %s", terrstr()); return -1; } @@ -732,7 +789,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, pSyncNode->vgId) != 0) { + if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { sRError(pReceiver, "snapshot receiver build rsp failed since %s", terrstr()); return -1; } @@ -869,6 +926,11 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // update sender pSender->snapshot = snapshot; + if (pMsg->payloadType == TAOS_SYNC_SNAP_INFO_DIFF) { + SMsgHead *msgHead = (void *)pMsg->data; + ASSERT(msgHead->vgId == pSyncNode->vgId); + pSender->snapshotParam.data = pMsg->data; + } // start reader int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); if (code != 0) { From 4d554884950565f45490ce8c2aacc47aac4f5187 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 12 Sep 2023 14:34:06 +0800 Subject: [PATCH 102/177] feat: use SSnapshotParam as a parameter to vnodeSnapReaderOpen --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 3 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 62 +++++++--------------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 9 +++- source/dnode/vnode/src/vnd/vnodeOpen.c | 14 ++--- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 10 ++-- source/dnode/vnode/src/vnd/vnodeSync.c | 3 +- 7 files changed, 41 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ba7bad67e4..baff19e3d4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -259,7 +259,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); // SVSnapReader -int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader); +int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader); void vnodeSnapReaderClose(SVSnapReader *pReader); int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData); // SVSnapWriter diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 902cc782ab..27a393abf4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -295,7 +295,8 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback); // STsdbSnapReader ======================================== -int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pExclude, +int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap); +int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges, STsdbSnapReader** ppReader); int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader); int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index b082c450f5..be0271135d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1252,23 +1252,12 @@ _out: return Z; } -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pExclude, +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, TSnapRangeArray **fsrArr) { int32_t code = -1; STFileSet *fset; STSnapRange *fsr1 = NULL; - TSnapRangeArray *snapF = tsdbFSToSnapRangeArray(fs); - if (snapF == NULL) { - tsdbError("failed to generate snap ranges from fs since %s.", terrstr()); - goto _out; - } - TSnapRangeArray *snapD = tsdbSnapDiff(snapF, pExclude); - if (snapD == NULL) { - tsdbError("failed to get diff of snap ranges since %s.", terrstr()); - goto _out; - } - fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); if (fsrArr[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1278,37 +1267,32 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev int32_t i = 0; code = 0; - // TODO: use the same fs fSetArr as get snapDiff. The following treatment is potentially wrong - // if the fSetArr are changed. taosThreadRwlockRdlock(&fs->tsdb->rwLock); TARRAY2_FOREACH(fs->fSetArr, fset) { - while (i < TARRAY2_SIZE(snapD)) { - STSnapRange *u = TARRAY2_GET(snapD, i); - if (fset->fid < u->fid) { - break; - } else if (fset->fid > u->fid) { - ASSERT(false); + int64_t sver1 = sver; + int64_t ever1 = ever; + + while (pRanges && i < TARRAY2_SIZE(pRanges)) { + STSnapRange *u = TARRAY2_GET(pRanges, i); + if (fset->fid > u->fid) { i++; continue; - } else { + } + + if (fset->fid == u->fid) { + sver1 = u->sver; i++; } - int64_t sver1 = TMAX(sver, u->sver); - int64_t ever1 = TMIN(ever, u->ever); - if (sver1 > ever1) { - continue; - } - tsdbInfo("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); - - code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); - if (code) break; - - code = TARRAY2_APPEND(fsrArr[0], fsr1); - if (code) break; - - fsr1 = NULL; } + tsdbInfo("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); + + code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); if (code) break; + + code = TARRAY2_APPEND(fsrArr[0], fsr1); + if (code) break; + + fsr1 = NULL; } taosThreadRwlockUnlock(&fs->tsdb->rwLock); @@ -1319,14 +1303,6 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev } _out: - TSnapRangeArray **ppArrs[2] = {&snapF, &snapD}; - int len = sizeof(ppArrs) / sizeof(ppArrs[0]); - for (int k = 0; k < len; k++) { - if (ppArrs[k][0] == NULL) continue; - TARRAY2_DESTROY(ppArrs[k][0], tsdbTSnapRangeClear); - taosMemoryFree(ppArrs[k][0]); - ppArrs[k][0] = NULL; - } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 2fdad2d662..be88f4e671 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -412,7 +412,7 @@ _exit: return code; } -int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, void* pExclude, +int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges, STsdbSnapReader** reader) { int32_t code = 0; int32_t lino = 0; @@ -425,7 +425,7 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pExclude, &reader[0]->fsrArr); + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1157,3 +1157,8 @@ _exit: } return code; } + +int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { + // TODO: get the full and diff info of tsdb Snap + return 0; +} diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 9228269992..fada83a7f1 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -518,13 +518,9 @@ void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } -int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { - pSnapshot->lastApplyIndex = pVnode->state.committed; - pSnapshot->lastApplyTerm = pVnode->state.commitTerm; - pSnapshot->lastConfigIndex = -1; - - if (pSnapshot->typ == TAOS_SYNC_SNAP_INFO_FULL) { - // TODO: get full info of snapshots - } - return 0; +int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { + pSnap->lastApplyIndex = pVnode->state.committed; + pSnap->lastApplyTerm = pVnode->state.commitTerm; + pSnap->lastConfigIndex = -1; + return tsdbSnapGetInfo(pVnode->pTsdb, pSnap); } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 6beb8e8dbf..5934826ea4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -29,7 +29,7 @@ struct SVSnapReader { SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; - TSnapRangeArray *pExclude; + TSnapRangeArray *pRanges; STsdbSnapReader *pTsdbReader; // tq int8_t tqHandleDone; @@ -48,8 +48,10 @@ struct SVSnapReader { SRSmaSnapReader *pRsmaReader; }; -int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) { +int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader) { int32_t code = 0; + int64_t sver = pParam->start; + int64_t ever = pParam->end; SVSnapReader *pReader = NULL; pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader)); @@ -61,7 +63,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe pReader->sver = sver; pReader->ever = ever; - // TODO: pReader->pEx + // TODO: decode pParam->data and store the result in pReader->pRanges vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; @@ -179,7 +181,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { - code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pExclude, + code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges, &pReader->pTsdbReader); if (code) goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b9f2d23c7b..e676219b11 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -475,8 +475,7 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { SVnode *pVnode = pFsm->data; - SSnapshotParam *pSnapshotParam = pParam; - int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader); + int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader); return code; } From 517f1f7e40b1ecb5ef086c0463e58a8a06f7a5ff Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 12 Sep 2023 19:27:54 +0800 Subject: [PATCH 103/177] fixup: fix syncNodeOnSnapshotPrep --- source/libs/sync/src/syncSnapshot.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 00dcd7e949..73b6940628 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -605,15 +605,16 @@ _SEND_REPLY: // build msg ; // make complier happy - code = -1; SSnapshot snapInfo = {.typ = TAOS_SYNC_SNAP_INFO_DIFF}; int32_t dataLen = 0; if (pMsg->dataLen > 0) { void *data = taosMemoryCalloc(1, pMsg->dataLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _out; } + dataLen = pMsg->dataLen; memcpy(data, pMsg->data, dataLen); snapInfo.data = data; data = NULL; @@ -627,6 +628,7 @@ _SEND_REPLY: SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) { sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); + code = terrno; goto _out; } @@ -650,10 +652,9 @@ _SEND_REPLY: syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver pre-snapshot"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); - goto _out; + code = terrno; } - code = 0; _out: if (snapInfo.data) { taosMemoryFree(snapInfo.data); From b9b62a9a005982aa908fe7636cd8bb0fcc4ef183 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 10:38:45 +0800 Subject: [PATCH 104/177] add rpc sync read timeout --- include/libs/transport/trpc.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 83dbf2c4af..accb7e6f24 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -163,6 +163,7 @@ int rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc in // These functions will not be called in the child process int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int32_t timeoutMs); int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); void *rpcAllocHandle(); void rpcSetIpWhite(void *thandl, void *arg); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3453731c03..7edd6d7d63 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -160,7 +160,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecv(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp); + rpcSendRecvWithTimeout(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp, 1000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[256]; From c1b2eedd3cdf2994a1e6f96ece68285ea20e8113 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 12 Sep 2023 19:29:22 +0800 Subject: [PATCH 105/177] feat: impl tsdbSnapGetInfo --- source/dnode/vnode/src/inc/tsdb.h | 5 + source/dnode/vnode/src/tsdb/tsdbFS2.c | 269 --------------------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 198 ++++++++++++++- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 20 +- 4 files changed, 220 insertions(+), 272 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4996c6c484..ffd74dc3d1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -676,6 +676,11 @@ typedef TARRAY2(STFileSet *) TFileSetArray; typedef struct STSnapRange STSnapRange; typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges +// util +int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); +int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); +void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); + struct STsdbReadSnap { SMemTable *pMem; SQueryNode *pNode; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index be0271135d..01d423da2c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -983,275 +983,6 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -static int32_t tsdbTSnapRangeCmprFn(STSnapRange *fsr1, STSnapRange *fsr2) { - if (fsr1->fid < fsr2->fid) return -1; - if (fsr1->fid > fsr2->fid) return 1; - if (fsr1->sver < fsr2->sver) return -1; - if (fsr1->sver > fsr2->sver) return 1; - if (fsr1->ever < fsr2->ever) return -1; - if (fsr1->ever < fsr2->ever) return 1; - return 0; -} - -static int32_t tsdbTFileInsertSnapRange(STFile *f, TSnapRangeArray *snapR) { - STSnapRange *fsr = taosMemoryCalloc(1, sizeof(*fsr)); - if (fsr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - fsr->fid = f->fid; - fsr->sver = f->minVer; - fsr->ever = f->maxVer; - - int32_t code = TARRAY2_SORT_INSERT(snapR, fsr, tsdbTSnapRangeCmprFn); - if (code) { - taosMemoryFree(fsr); - fsr = NULL; - } - return code; -} - -static int32_t tsdbTFSetInsertSnapRange(STFileSet *fset, TSnapRangeArray *snapR) { - STFile tf = {.fid = fset->fid, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; - for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset->farr[ftype] == NULL) continue; - STFile *f = fset->farr[ftype]->f; - tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = f->minVer, .maxVer = f->maxVer}); - } - - int32_t code = tsdbTFileInsertSnapRange(&tf, snapR); - if (code) return code; - - const SSttLvl *lvl; - TARRAY2_FOREACH(fset->lvlArr, lvl) { - STFileObj *fobj; - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - // tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = fobj->f->minVer, .maxVer = fobj->f->maxVer}); - code = tsdbTFileInsertSnapRange(fobj->f, snapR); - if (code) return code; - } - } - - // int32_t code = tsdbTFileInsertSnapRange(&tf, snapR); - // if (code) return code; - return code; -} - -TSnapRangeArray *tsdbFSToSnapRangeArray(STFileSystem *fs) { - int32_t code = 0; - TSnapRangeArray *snapR = taosMemoryCalloc(1, sizeof(*snapR)); - if (snapR == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - TARRAY2_INIT(snapR); - - taosThreadRwlockRdlock(&fs->tsdb->rwLock); - STFileSet *fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - code = tsdbTFSetInsertSnapRange(fset, snapR); - if (code) break; - } - taosThreadRwlockUnlock(&fs->tsdb->rwLock); - - if (code) { - TARRAY2_DESTROY(snapR, tsdbTSnapRangeClear); - taosMemoryFree(snapR); - snapR = NULL; - } - return snapR; -} - -static STSnapRange *taosDupSnapRange(STSnapRange *x) { - STSnapRange *y = taosMemoryCalloc(1, sizeof(STSnapRange)); - if (y == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - ASSERTS(terrno == 0, "Out of memory"); - return NULL; - } - y->fid = x->fid; - y->sver = x->sver; - y->ever = x->ever; - return y; -} - -static TSnapRangeArray *taosDupSnapRangeArray(const TSnapRangeArray *X) { - TSnapRangeArray *Y = taosMemoryCalloc(1, sizeof(*Y)); - if (Y == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - TARRAY2_INIT(Y); - - if (X) { - STSnapRange *x; - TARRAY2_FOREACH(X, x) { - STSnapRange *tp = taosDupSnapRange(x); - TARRAY2_APPEND(Y, tp); - } - } - return Y; -} - -static TSnapRangeArray *tsdbSnapDiff(const TSnapRangeArray *snapR, const TSnapRangeArray *pExclude) { - TSnapRangeArray *Z = NULL; - TSnapRangeArray *U = NULL; - TSnapRangeArray *V = NULL; - TSnapRangeArray *X = taosDupSnapRangeArray(snapR); - TSnapRangeArray *Y = taosDupSnapRangeArray(pExclude); - int32_t code = -1; - - // separate intersections of snap ranges - U = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); - if (U == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _out; - } - TARRAY2_INIT(U); - - V = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); - if (V == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _out; - } - TARRAY2_INIT(V); - - int32_t i = 0; - int32_t j = 0; - while (i < TARRAY2_SIZE(X) && j < TARRAY2_SIZE(Y)) { - STSnapRange *x = TARRAY2_GET(X, i); - STSnapRange *y = TARRAY2_GET(Y, j); - - if (x->fid < y->fid) { - STSnapRange *tmp = taosDupSnapRange(x); - TARRAY2_APPEND(U, tmp); - i++; - } else if (x->fid > y->fid) { - STSnapRange *tmp = taosDupSnapRange(y); - TARRAY2_APPEND(V, tmp); - j++; - } else { - if (x->sver < y->sver) { - if (x->ever < y->ever) { - STSnapRange *tmp = taosDupSnapRange(x); - TARRAY2_APPEND(U, tmp); - i++; - } else { - STSnapRange *tmp = taosDupSnapRange(x); - tmp->ever = y->sver - 1; - TARRAY2_APPEND(U, tmp); - x->sver = y->sver; - } - } else if (x->sver > y->sver) { - if (y->ever < x->ever) { - STSnapRange *tmp = taosDupSnapRange(y); - TARRAY2_APPEND(V, tmp); - j++; - } else { - STSnapRange *tmp = taosDupSnapRange(y); - tmp->ever = x->sver - 1; - TARRAY2_APPEND(V, tmp); - y->sver = x->sver; - } - } else { - if (x->ever < y->ever) { - STSnapRange *tmp = taosDupSnapRange(x); - TARRAY2_APPEND(U, tmp); - i++; - tmp = taosDupSnapRange(y); - tmp->ever = x->ever; - TARRAY2_APPEND(V, tmp); - y->sver = x->ever + 1; - } else if (x->ever > y->ever) { - STSnapRange *tmp = taosDupSnapRange(y); - TARRAY2_APPEND(V, tmp); - j++; - tmp = taosDupSnapRange(x); - tmp->ever = y->ever; - TARRAY2_APPEND(U, tmp); - x->sver = y->ever + 1; - } else { - STSnapRange *tmp = taosDupSnapRange(x); - TARRAY2_APPEND(U, tmp); - i++; - tmp = taosDupSnapRange(y); - TARRAY2_APPEND(V, tmp); - j++; - } - } - } - } - while (i < TARRAY2_SIZE(X)) { - STSnapRange *x = TARRAY2_GET(X, i); - STSnapRange *tmp = taosDupSnapRange(x); - TARRAY2_APPEND(U, tmp); - i++; - } - while (j < TARRAY2_SIZE(Y)) { - STSnapRange *y = TARRAY2_GET(Y, j); - STSnapRange *tmp = taosDupSnapRange(y); - TARRAY2_APPEND(V, tmp); - j++; - } - - // difference of snap ranges - Z = taosMemoryCalloc(1, sizeof(*Z)); - if (Z == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _out; - } - TARRAY2_INIT(Z); - - i = 0; - j = 0; - while (i < TARRAY2_SIZE(U) && j < TARRAY2_SIZE(V)) { - STSnapRange *u = TARRAY2_GET(U, i); - STSnapRange *v = TARRAY2_GET(V, j); - - if (u->fid < v->fid) { - STSnapRange *tmp = taosDupSnapRange(u); - TARRAY2_APPEND(Z, tmp); - i++; - } else if (u->fid == v->fid) { - if (u->sver < v->sver) { - STSnapRange *tmp = taosDupSnapRange(u); - TARRAY2_APPEND(Z, tmp); - i++; - } else if (u->sver > v->sver) { - ASSERT(u->ever > v->ever); - j++; - } else { - ASSERT(u->ever == v->ever); - i++; - j++; - } - } - } - while (i < TARRAY2_SIZE(U)) { - STSnapRange *u = TARRAY2_GET(U, i); - STSnapRange *tmp = taosDupSnapRange(u); - TARRAY2_APPEND(Z, tmp); - i++; - } - - code = 0; -_out: - TSnapRangeArray **ppArrs[4] = {&X, &Y, &U, &V}; - int len = sizeof(ppArrs) / sizeof(ppArrs[0]); - for (int k = 0; k < len; k++) { - if (ppArrs[k][0] == NULL) continue; - TARRAY2_DESTROY(ppArrs[k][0], tsdbTSnapRangeClear); - taosMemoryFree(ppArrs[k][0]); - ppArrs[k][0] = NULL; - } - if (code != 0 && Z) { - TARRAY2_DESTROY(Z, tsdbTSnapRangeClear); - taosMemoryFree(Z); - Z = NULL; - } - return Z; -} - int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, TSnapRangeArray **fsrArr) { int32_t code = -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index be88f4e671..de1ea27c4d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1158,7 +1158,201 @@ _exit: return code; } -int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { - // TODO: get the full and diff info of tsdb Snap +static int32_t tsdbTSnapRangeCmprFn(STSnapRange* fsr1, STSnapRange* fsr2) { + if (fsr1->fid < fsr2->fid) return -1; + if (fsr1->fid > fsr2->fid) return 1; return 0; } + +static int32_t tsdbTFileInsertSnapRange(STFile* f, TSnapRangeArray* snapR) { + STSnapRange* fsr = taosMemoryCalloc(1, sizeof(*fsr)); + if (fsr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + fsr->fid = f->fid; + fsr->sver = f->minVer; + fsr->ever = f->maxVer; + + int32_t code = TARRAY2_SORT_INSERT(snapR, fsr, tsdbTSnapRangeCmprFn); + if (code) { + taosMemoryFree(fsr); + fsr = NULL; + } + return code; +} + +static int32_t tsdbTFSetInsertSnapRange(STFileSet* fset, TSnapRangeArray* snapR) { + STFile tf = {.fid = fset->fid, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + STFile* f = fset->farr[ftype]->f; + tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = f->minVer, .maxVer = f->maxVer}); + } + + const SSttLvl* lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = fobj->f->minVer, .maxVer = fobj->f->maxVer}); + } + } + + int32_t code = tsdbTFileInsertSnapRange(&tf, snapR); + if (code) return code; + return code; +} + +static TSnapRangeArray* tsdbGetSnapRangeArray(STFileSystem* fs) { + int32_t code = 0; + TSnapRangeArray* snapR = taosMemoryCalloc(1, sizeof(*snapR)); + if (snapR == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + TARRAY2_INIT(snapR); + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + STFileSet* fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTFSetInsertSnapRange(fset, snapR); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(snapR, tsdbTSnapRangeClear); + taosMemoryFree(snapR); + snapR = NULL; + } + return snapR; +} + +int32_t tSerializeSnapRangeArray(void* buf, int32_t bufLen, TSnapRangeArray* pSnapR) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int8_t msgVer = 1; + int32_t arrLen = TARRAY2_SIZE(pSnapR); + int8_t reserved8 = 0; + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + if (tEncodeI8(&encoder, reserved8) < 0) goto _err; + if (tEncodeI32(&encoder, arrLen) < 0) goto _err; + + int64_t reserved64 = 0; + for (int32_t i = 0; i < arrLen; i++) { + STSnapRange* u = TARRAY2_GET(pSnapR, i); + int64_t fid = u->fid; + if (tEncodeI64(&encoder, fid) < 0) goto _err; + if (tEncodeI64(&encoder, u->sver) < 0) goto _err; + if (tEncodeI64(&encoder, u->ever) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + } + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeSnapRangeArray(void* buf, int32_t bufLen, TSnapRangeArray* pSnapR) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int8_t msgVer = 0; + int32_t arrLen = 0; + int8_t reserved8 = 0; + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; + if (tDecodeI32(&decoder, &arrLen) < 0) goto _err; + + int64_t fid = 0; + int64_t reserved64 = 0; + STSnapRange* pRange = NULL; + for (int32_t i = 0; i < arrLen; i++) { + pRange = taosMemoryCalloc(1, sizeof(STSnapRange)); + if (tDecodeI64(&decoder, &fid) < 0) goto _err; + pRange->fid = fid; + if (tDecodeI64(&decoder, &pRange->sver) < 0) goto _err; + if (tDecodeI64(&decoder, &pRange->ever) < 0) goto _err; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + TARRAY2_APPEND(pSnapR, pRange); + pRange = NULL; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + if (pRange) { + taosMemoryFree(pRange); + pRange = NULL; + } + tDecoderClear(&decoder); + return -1; +} + +void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { + TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); + taosMemoryFree(ppSnap[0]); + ppSnap[0] = NULL; +} + +static int32_t tsdbSnapInfoDataLenCalc(TSnapRangeArray* pSnap) { + int32_t headerLen = 8; + int32_t itemLen = sizeof(STSnapRange) + 8; + int32_t size = TARRAY2_SIZE(pSnap); + return headerLen + itemLen * size; +} + +int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { + int32_t code = 0; + if (pSnap->typ == TAOS_SYNC_SNAP_INFO_BRIEF) { + return 0; + } + code = -1; + TSnapRangeArray* snapR = tsdbGetSnapRangeArray(pTsdb->pFS); + if (snapR == NULL) { + goto _out; + } + if (pSnap->typ == TAOS_SYNC_SNAP_INFO_DIFF) { + for (int32_t i = 0; i < TARRAY2_SIZE(snapR); i++) { + STSnapRange* u = TARRAY2_GET(snapR, i); + u->sver = u->ever + 1; + u->ever = VERSION_MAX; + } + } + + int32_t bufLen = sizeof(SMsgHead) + tsdbSnapInfoDataLenCalc(snapR); + void* data = taosMemoryRealloc(pSnap->data, bufLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + pSnap->data = data; + void* buf = ((char*)data) + sizeof(SMsgHead); + int32_t tlen = 0; + + if ((tlen = tSerializeSnapRangeArray(buf, bufLen, snapR)) < 0) { + tsdbError("vgId:%d, failed to serialize snap range since %s", TD_VID(pTsdb->pVnode), terrstr()); + goto _out; + } + SMsgHead* msgHead = pSnap->data; + msgHead->contLen = tlen; + msgHead->vgId = TD_VID(pTsdb->pVnode); + + code = 0; +_out: + if (snapR) { + tsdbSnapRangeArrayDestroy(&snapR); + } + + return code; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 5934826ea4..418d8632c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -63,7 +63,22 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->sver = sver; pReader->ever = ever; - // TODO: decode pParam->data and store the result in pReader->pRanges + if (pParam->data) { + pReader->pRanges = taosMemoryCalloc(1, sizeof(*pReader->pRanges)); + if (pReader->pRanges == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pReader->pRanges); + SMsgHead *msgHead = pParam->data; + ASSERT(msgHead->vgId == TD_VID(pVnode)); + void *buf = (char *)pParam->data + sizeof(SMsgHead); + + if (tDeserializeSnapRangeArray(buf, msgHead->contLen, pReader->pRanges) < 0) { + vError("vgId:%d, failed to deserialize snap range.", TD_VID(pVnode)); + goto _err; + } + } vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; @@ -101,6 +116,9 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader); } + if (pReader->pRanges) { + tsdbSnapRangeArrayDestroy(&pReader->pRanges); + } taosMemoryFree(pReader); } From f99795d027f8a1ed95abed40787735bb321cbfe5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 13 Sep 2023 09:16:08 +0800 Subject: [PATCH 106/177] enh: use tsdbSnapRangeArrayDestroy --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 9 --------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 12 +++++++----- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 01d423da2c..d71473b079 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1037,15 +1037,6 @@ _out: return code; } -int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr) { - if (fsrArr[0]) { - TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); - taosMemoryFreeClear(fsrArr[0]); - fsrArr[0] = NULL; - } - return 0; -} - const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; static int32_t tsdbFSRunBgTask(void *arg) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index de1ea27c4d..3e18f01f04 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -432,7 +432,7 @@ _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); + tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -460,7 +460,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); + tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -1300,9 +1300,11 @@ _err: } void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { - TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); - taosMemoryFree(ppSnap[0]); - ppSnap[0] = NULL; + if (ppSnap && ppSnap[0]) { + TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); + taosMemoryFree(ppSnap[0]); + ppSnap[0] = NULL; + } } static int32_t tsdbSnapInfoDataLenCalc(TSnapRangeArray* pSnap) { From 410ced8320c554fffec525c78bb35ffb05968a65 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 14 Sep 2023 18:22:19 +0800 Subject: [PATCH 107/177] feat: use TLV format to encode data of snapshot info --- include/common/tmsgdef.h | 4 +- include/libs/sync/sync.h | 6 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 4 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 27 ++ source/dnode/vnode/src/tsdb/tsdbFSet2.h | 1 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 370 ++++++++++++++------ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 46 ++- source/libs/sync/src/syncMessage.c | 36 -- source/libs/sync/src/syncSnapshot.c | 41 ++- 10 files changed, 371 insertions(+), 168 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 4a2ae18765..b92bba831c 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -299,8 +299,8 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_LOCAL_CMD, "sync-local-cmd", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT, "sync-pre-snapshot", NULL, NULL) // no longer used - TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT_REPLY, "sync-pre-snapshot-reply", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT, "sync-prep-snapshot", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 53e6ec0d71..cc381bb54e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -101,6 +101,12 @@ typedef struct SNodeInfo { ESyncRole nodeRole; } SNodeInfo; +typedef struct SSyncTLV { + int32_t typ; + int32_t len; + char val[]; +} SSyncTLV; + typedef struct SSyncCfg { int32_t totalReplicaNum; int32_t replicaNum; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 24b5b2566c..d5488da770 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -238,7 +238,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -246,7 +246,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 0e17d2b75f..b4fe824466 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -848,14 +848,14 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ffd74dc3d1..4cea7c5e85 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -681,6 +681,33 @@ int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSn int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); +// snap partition list +typedef TARRAY2(SVersionRange) SVerRangeList; +typedef struct STsdbSnapPartition STsdbSnapPartition; +typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList; +// util +STsdbSnapPartList *tsdbSnapPartListCreate(); +void tsdbSnapPartListDestroy(STsdbSnapPartList **ppList); +int32_t tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); +int32_t tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges); + +enum { + TSDB_SNAP_RANGE_TYP_HEAD = 0, + TSDB_SNAP_RANGE_TYP_DATA, + TSDB_SNAP_RANGE_TYP_SMA, + TSDB_SNAP_RANGE_TYP_TOMB, + TSDB_SNAP_RANGE_TYP_STT, + TSDB_SNAP_RANGE_TYP_MAX, +}; + +struct STsdbSnapPartition { + int64_t fid; + int8_t stat; + SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; +}; + +// snap read struct STsdbReadSnap { SMemTable *pMem; SQueryNode *pNode; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 756250157b..e6d78a8cfe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -81,6 +81,7 @@ struct SSttLvl { struct STFileSet { int32_t fid; + int8_t stat; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array }; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 3e18f01f04..9710eee6c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1158,96 +1158,198 @@ _exit: return code; } -static int32_t tsdbTSnapRangeCmprFn(STSnapRange* fsr1, STSnapRange* fsr2) { - if (fsr1->fid < fsr2->fid) return -1; - if (fsr1->fid > fsr2->fid) return 1; +// snap part +static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; return 0; } -static int32_t tsdbTFileInsertSnapRange(STFile* f, TSnapRangeArray* snapR) { - STSnapRange* fsr = taosMemoryCalloc(1, sizeof(*fsr)); - if (fsr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - fsr->fid = f->fid; - fsr->sver = f->minVer; - fsr->ever = f->maxVer; - - int32_t code = TARRAY2_SORT_INSERT(snapR, fsr, tsdbTSnapRangeCmprFn); - if (code) { - taosMemoryFree(fsr); - fsr = NULL; - } - return code; +static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { + if (x->minVer < y->minVer) return -1; + if (x->minVer > y->minVer) return 1; + if (x->maxVer < y->maxVer) return -1; + if (x->maxVer > y->maxVer) return 1; + return 0; } -static int32_t tsdbTFSetInsertSnapRange(STFileSet* fset, TSnapRangeArray* snapR) { - STFile tf = {.fid = fset->fid, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; - for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset->farr[ftype] == NULL) continue; - STFile* f = fset->farr[ftype]->f; - tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = f->minVer, .maxVer = f->maxVer}); +STsdbSnapPartition* tsdbSnapPartitionCreate() { + STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); + if (pSP == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + TARRAY2_INIT(&pSP->verRanges[i]); + } + return pSP; +} + +void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { + if (ppSP == NULL || ppSP[0] == NULL) { + return; + } + for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); + } + taosMemoryFree(ppSP[0]); + ppSP[0] = NULL; +} + +static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { + switch (ftype) { + case TSDB_FTYPE_HEAD: + return TSDB_SNAP_RANGE_TYP_HEAD; + case TSDB_FTYPE_DATA: + return TSDB_SNAP_RANGE_TYP_DATA; + case TSDB_FTYPE_SMA: + return TSDB_SNAP_RANGE_TYP_SMA; + case TSDB_FTYPE_TOMB: + return TSDB_SNAP_RANGE_TYP_TOMB; + case TSDB_FTYPE_STT: + return TSDB_SNAP_RANGE_TYP_STT; + } + return TSDB_SNAP_RANGE_TYP_MAX; +} + +static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { + STsdbSnapPartition* p = tsdbSnapPartitionCreate(); + if (p == NULL) { + goto _err; } + int32_t typ = 0; + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + typ = tsdbFTypeToSRangeTyp(ftype); + ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); + STFile* f = fset->farr[ftype]->f; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + } + + typ = TSDB_SNAP_RANGE_TYP_STT; const SSttLvl* lvl; TARRAY2_FOREACH(fset->lvlArr, lvl) { STFileObj* fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { - tsdbTFileUpdVerRange(&tf, (SVersionRange){.minVer = fobj->f->minVer, .maxVer = fobj->f->maxVer}); + STFile* f = fobj->f; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); } } + ppSP[0] = p; + return 0; - int32_t code = tsdbTFileInsertSnapRange(&tf, snapR); - if (code) return code; - return code; +_err: + tsdbSnapPartitionClear(&p); + return -1; } -static TSnapRangeArray* tsdbGetSnapRangeArray(STFileSystem* fs) { - int32_t code = 0; - TSnapRangeArray* snapR = taosMemoryCalloc(1, sizeof(*snapR)); - if (snapR == NULL) { +STsdbSnapPartList* tsdbSnapPartListCreate() { + STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); + if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - TARRAY2_INIT(snapR); + TARRAY2_INIT(pList); + return pList; +} +static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { + STsdbSnapPartList* pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + return NULL; + } + + int32_t code = 0; taosThreadRwlockRdlock(&fs->tsdb->rwLock); STFileSet* fset; TARRAY2_FOREACH(fs->fSetArr, fset) { - code = tsdbTFSetInsertSnapRange(fset, snapR); - if (code) break; + STsdbSnapPartition* pItem = NULL; + if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { + code = -1; + break; + } + ASSERT(pItem != NULL); + TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); } taosThreadRwlockUnlock(&fs->tsdb->rwLock); if (code) { - TARRAY2_DESTROY(snapR, tsdbTSnapRangeClear); - taosMemoryFree(snapR); - snapR = NULL; + TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); + taosMemoryFree(pList); + pList = NULL; } - return snapR; + return pList; } -int32_t tSerializeSnapRangeArray(void* buf, int32_t bufLen, TSnapRangeArray* pSnapR) { +int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + hdrLen += sizeof(msgVer); + hdrLen += sizeof(len); + datLen += hdrLen; + + for (int32_t u = 0; u < len; u++) { + STsdbSnapPartition* p = TARRAY2_GET(pList, u); + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t uItem = 0; + uItem += sizeof(STsdbSnapPartition); + uItem += sizeof(typMax); + + for (int32_t i = 0; i < typMax; i++) { + int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); + int32_t jItem = 0; + jItem += sizeof(SVersionRange); + jItem += sizeof(int64_t); + uItem += sizeof(iLen) + jItem * iLen; + } + datLen += uItem; + } + return datLen; +} + +int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); - int8_t msgVer = 1; - int32_t arrLen = TARRAY2_SIZE(pSnapR); int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + if (tStartEncode(&encoder) < 0) goto _err; if (tEncodeI8(&encoder, msgVer) < 0) goto _err; - if (tEncodeI8(&encoder, reserved8) < 0) goto _err; - if (tEncodeI32(&encoder, arrLen) < 0) goto _err; + if (tEncodeI32(&encoder, len) < 0) goto _err; - int64_t reserved64 = 0; - for (int32_t i = 0; i < arrLen; i++) { - STSnapRange* u = TARRAY2_GET(pSnapR, i); - int64_t fid = u->fid; - if (tEncodeI64(&encoder, fid) < 0) goto _err; - if (tEncodeI64(&encoder, u->sver) < 0) goto _err; - if (tEncodeI64(&encoder, u->ever) < 0) goto _err; - if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + for (int32_t u = 0; u < len; u++) { + STsdbSnapPartition* p = TARRAY2_GET(pList, u); + if (tEncodeI64(&encoder, p->fid) < 0) goto _err; + if (tEncodeI8(&encoder, p->stat) < 0) goto _err; + if (tEncodeI8(&encoder, reserved8) < 0) goto _err; + if (tEncodeI16(&encoder, reserved16) < 0) goto _err; + + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + if (tEncodeI32(&encoder, typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = TARRAY2_SIZE(iList); + + if (tEncodeI32(&encoder, iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = TARRAY2_GET(iList, j); + if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; + if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + } + } } tEndEncode(&encoder); @@ -1260,30 +1362,47 @@ _err: return -1; } -int32_t tDeserializeSnapRangeArray(void* buf, int32_t bufLen, TSnapRangeArray* pSnapR) { +int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); - int8_t msgVer = 0; - int32_t arrLen = 0; int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + STsdbSnapPartition* p = NULL; + + int8_t msgVer = 0; + int32_t len = 0; if (tStartDecode(&decoder) < 0) goto _err; if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; - if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; - if (tDecodeI32(&decoder, &arrLen) < 0) goto _err; + if (tDecodeI32(&decoder, &len) < 0) goto _err; - int64_t fid = 0; - int64_t reserved64 = 0; - STSnapRange* pRange = NULL; - for (int32_t i = 0; i < arrLen; i++) { - pRange = taosMemoryCalloc(1, sizeof(STSnapRange)); - if (tDecodeI64(&decoder, &fid) < 0) goto _err; - pRange->fid = fid; - if (tDecodeI64(&decoder, &pRange->sver) < 0) goto _err; - if (tDecodeI64(&decoder, &pRange->ever) < 0) goto _err; - if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; - TARRAY2_APPEND(pSnapR, pRange); - pRange = NULL; + for (int32_t u = 0; u < len; u++) { + p = tsdbSnapPartitionCreate(); + if (p == NULL) goto _err; + if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; + if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; + if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; + if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; + + int32_t typMax = 0; + if (tDecodeI32(&decoder, &typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = 0; + if (tDecodeI32(&decoder, &iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = {0}; + if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; + if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + TARRAY2_APPEND(iList, r); + } + } + TARRAY2_APPEND(pList, p); + p = NULL; } tEndDecode(&decoder); @@ -1291,14 +1410,52 @@ int32_t tDeserializeSnapRangeArray(void* buf, int32_t bufLen, TSnapRangeArray* p return 0; _err: - if (pRange) { - taosMemoryFree(pRange); - pRange = NULL; + if (p) { + tsdbSnapPartitionClear(&p); } tDecoderClear(&decoder); return -1; } +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) { + TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); + if (pDiff == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pDiff); + + STsdbSnapPartition* part; + TARRAY2_FOREACH(pList, part) { + STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange)); + if (r == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int64_t ever = -1; + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &part->verRanges[i]; + SVersionRange r = {0}; + TARRAY2_FOREACH(iList, r) { + if (r.maxVer < r.minVer) continue; + ever = TMAX(ever, r.maxVer); + } + } + r->sver = ever + 1; + r->ever = VERSION_MAX; + TARRAY2_APPEND(pDiff, r); + } + ppRanges[0] = pDiff; + return 0; + +_err: + if (pDiff) { + tsdbSnapRangeArrayDestroy(&pDiff); + } + return -1; +} + void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { if (ppSnap && ppSnap[0]) { TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); @@ -1307,53 +1464,64 @@ void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { } } -static int32_t tsdbSnapInfoDataLenCalc(TSnapRangeArray* pSnap) { - int32_t headerLen = 8; - int32_t itemLen = sizeof(STSnapRange) + 8; - int32_t size = TARRAY2_SIZE(pSnap); - return headerLen + itemLen * size; +void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { + if (ppList == NULL || ppList[0] == NULL) return; + + TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); + taosMemoryFree(ppList[0]); + ppList[0] = NULL; } int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { - int32_t code = 0; - if (pSnap->typ == TAOS_SYNC_SNAP_INFO_BRIEF) { + if (pSnap->typ != TDMT_SYNC_PREP_SNAPSHOT && pSnap->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { return 0; } - code = -1; - TSnapRangeArray* snapR = tsdbGetSnapRangeArray(pTsdb->pFS); - if (snapR == NULL) { - goto _out; - } - if (pSnap->typ == TAOS_SYNC_SNAP_INFO_DIFF) { - for (int32_t i = 0; i < TARRAY2_SIZE(snapR); i++) { - STSnapRange* u = TARRAY2_GET(snapR, i); - u->sver = u->ever + 1; - u->ever = VERSION_MAX; - } + int code = -1; + STsdbSnapPartList* pList = tsdbGetSnapPartList(pTsdb->pFS); + if (pList == NULL) goto _out; + + if (pSnap->typ == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { } - int32_t bufLen = sizeof(SMsgHead) + tsdbSnapInfoDataLenCalc(snapR); - void* data = taosMemoryRealloc(pSnap->data, bufLen); + void* buf = NULL; + int32_t tlen = 0; + // estimate data length encode + int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY + bufLen += sizeof(SSyncTLV); // subtyp: SNAP_DATA_TSDB + bufLen += tTsdbSnapPartListDataLenCalc(pList); + + void* data = taosMemoryRealloc(pSnap->data, bufLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _out; } pSnap->data = data; - void* buf = ((char*)data) + sizeof(SMsgHead); - int32_t tlen = 0; - if ((tlen = tSerializeSnapRangeArray(buf, bufLen, snapR)) < 0) { + // header + SSyncTLV* datHead = (void*)pSnap->data; + datHead->typ = pSnap->typ; + datHead->len = 0; + + // tsdb + SSyncTLV* tsdbHead = (void*)datHead->val; + tsdbHead->typ = SNAP_DATA_TSDB; + + buf = tsdbHead->val; + tlen = 0; + if ((tlen = tSerializeTsdbSnapPartList(buf, bufLen, pList)) < 0) { tsdbError("vgId:%d, failed to serialize snap range since %s", TD_VID(pTsdb->pVnode), terrstr()); goto _out; } - SMsgHead* msgHead = pSnap->data; - msgHead->contLen = tlen; - msgHead->vgId = TD_VID(pTsdb->pVnode); + tsdbHead->len = tlen; + datHead->len += sizeof(SSyncTLV) + tsdbHead->len; + + // rsma code = 0; + _out: - if (snapR) { - tsdbSnapRangeArrayDestroy(&snapR); + if (pList) { + tsdbSnapPartListDestroy(&pList); } return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 418d8632c9..c56644ca2c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -45,6 +45,7 @@ struct SVSnapReader { SStreamStateReader *pStreamStateReader; // rsma int8_t rsmaDone; + TSnapRangeArray *pRsmaRanges; SRSmaSnapReader *pRsmaReader; }; @@ -64,19 +65,44 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->ever = ever; if (pParam->data) { - pReader->pRanges = taosMemoryCalloc(1, sizeof(*pReader->pRanges)); - if (pReader->pRanges == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + SSyncTLV *datHead = (void *)pParam->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + terrno = TSDB_CODE_INVALID_DATA_FMT; goto _err; } - TARRAY2_INIT(pReader->pRanges); - SMsgHead *msgHead = pParam->data; - ASSERT(msgHead->vgId == TD_VID(pVnode)); - void *buf = (char *)pParam->data + sizeof(SMsgHead); - if (tDeserializeSnapRangeArray(buf, msgHead->contLen, pReader->pRanges) < 0) { - vError("vgId:%d, failed to deserialize snap range.", TD_VID(pVnode)); - goto _err; + int32_t offset = 0; + while (offset + sizeof(SSyncTLV) < datHead->len) { + SSyncTLV *sectHead = (void *)(datHead->val + offset); + offset += sizeof(SSyncTLV) + sectHead->len; + void *buf = sectHead->val; + int32_t bufLen = sectHead->len; + ASSERT(sectHead->typ == SNAP_DATA_TSDB || sectHead->typ == SNAP_DATA_RSMA1); + STsdbSnapPartList *pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _err; + } + TSnapRangeArray **ppRanges = NULL; + if (sectHead->typ == SNAP_DATA_TSDB) { + ppRanges = &pReader->pRanges; + } else if (sectHead->typ == SNAP_DATA_RSMA1) { + ppRanges = &pReader->pRsmaRanges; + } + if (ppRanges == NULL) { + tsdbSnapPartListDestroy(&pList); + continue; + } + if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + tsdbSnapPartListDestroy(&pList); + goto _err; + } + tsdbSnapPartListDestroy(&pList); } } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 00ca6d8f90..9e035f60c2 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -216,42 +216,6 @@ int32_t syncBuildHeartbeatReply(SRpcMsg* pMsg, int32_t vgId) { return 0; } -#if 0 -int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId) { - int32_t bytes = sizeof(SyncPreSnapshot); - pMsg->pCont = rpcMallocCont(bytes); - pMsg->msgType = TDMT_SYNC_PRE_SNAPSHOT; - pMsg->contLen = bytes; - if (pMsg->pCont == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - SyncPreSnapshot* pPreSnapshot = pMsg->pCont; - pPreSnapshot->bytes = bytes; - pPreSnapshot->msgType = TDMT_SYNC_PRE_SNAPSHOT; - pPreSnapshot->vgId = vgId; - return 0; -} - -int32_t syncBuildPreSnapshotReply(SRpcMsg* pMsg, int32_t vgId) { - int32_t bytes = sizeof(SyncPreSnapshotReply); - pMsg->pCont = rpcMallocCont(bytes); - pMsg->msgType = TDMT_SYNC_PRE_SNAPSHOT_REPLY; - pMsg->contLen = bytes; - if (pMsg->pCont == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - SyncPreSnapshotReply* pPreSnapshotReply = pMsg->pCont; - pPreSnapshotReply->bytes = bytes; - pPreSnapshotReply->msgType = TDMT_SYNC_PRE_SNAPSHOT_REPLY; - pPreSnapshotReply->vgId = vgId; - return 0; -} -#endif - int32_t syncBuildSnapshotSend(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId) { int32_t bytes = sizeof(SyncSnapshotSend) + dataLen; pMsg->pCont = rpcMallocCont(bytes); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 73b6940628..681d256ec9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -98,7 +98,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { // Get full snapshot info SSyncNode *pSyncNode = pSender->pSyncNode; - SSnapshot snapInfo = {.typ = TAOS_SYNC_SNAP_INFO_FULL}; + SSnapshot snapInfo = {.typ = TDMT_SYNC_PREP_SNAPSHOT}; if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) { sSError(pSender, "snapshot get info failure since %s", terrstr()); goto _out; @@ -106,12 +106,15 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { int dataLen = 0; if (snapInfo.data) { - SMsgHead *msgHead = snapInfo.data; - ASSERT(msgHead->vgId == pSyncNode->vgId); - dataLen = sizeof(SMsgHead) + msgHead->contLen; + SSyncTLV *datHead = snapInfo.data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT) { + sSError(pSender, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + dataLen = sizeof(SSyncTLV) + datHead->len; } - // build begin msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSend(&rpcMsg, dataLen, pSender->pSyncNode->vgId) != 0) { sSError(pSender, "snapshot sender build msg failed since %s", terrstr()); @@ -605,7 +608,7 @@ _SEND_REPLY: // build msg ; // make complier happy - SSnapshot snapInfo = {.typ = TAOS_SYNC_SNAP_INFO_DIFF}; + SSnapshot snapInfo = {.typ = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; if (pMsg->dataLen > 0) { void *data = taosMemoryCalloc(1, pMsg->dataLen); @@ -614,15 +617,18 @@ _SEND_REPLY: code = terrno; goto _out; } - dataLen = pMsg->dataLen; - memcpy(data, pMsg->data, dataLen); + memcpy(data, pMsg->data, pMsg->dataLen); snapInfo.data = data; data = NULL; pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); - SMsgHead *msgHead = snapInfo.data; - ASSERT(msgHead->vgId == pSyncNode->vgId); - dataLen = msgHead->contLen; + SSyncTLV *datHead = snapInfo.data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); + code = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + dataLen = sizeof(SSyncTLV) + datHead->len; } SRpcMsg rpcMsg = {0}; @@ -927,12 +933,17 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // update sender pSender->snapshot = snapshot; - if (pMsg->payloadType == TAOS_SYNC_SNAP_INFO_DIFF) { - SMsgHead *msgHead = (void *)pMsg->data; - ASSERT(msgHead->vgId == pSyncNode->vgId); + // start reader + if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + SSyncTLV *datHead = (void *)pMsg->data; + if (datHead->typ != pMsg->payloadType) { + sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); + terrno = TSDB_CODE_INVALID_DATA_FMT; + return -1; + } pSender->snapshotParam.data = pMsg->data; } - // start reader + int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); if (code != 0) { sSError(pSender, "prepare snapshot failed since %s", terrstr()); From e1c03118abe8142ab48b15d8ddc8cfd20c4aa477 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 15 Sep 2023 19:58:48 +0800 Subject: [PATCH 108/177] enh: extract snapshot info for both of snap reader and writer --- include/libs/sync/sync.h | 6 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 106 +++++++++++++++------ source/dnode/vnode/src/vnd/vnodeSync.c | 5 +- source/libs/sync/src/syncSnapshot.c | 24 ++++- 5 files changed, 108 insertions(+), 35 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index cc381bb54e..50e60d2ef4 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -151,12 +151,12 @@ typedef struct SReConfigCbMeta { typedef struct SSnapshotParam { SyncIndex start; SyncIndex end; - void* data; // with SMsgHead + SSyncTLV* data; } SSnapshotParam; typedef struct SSnapshot { - ESyncSnapInfoTyp typ; - void* data; // with SMsgHead + int32_t typ; + SSyncTLV* data; SyncIndex lastApplyIndex; SyncTerm lastApplyTerm; SyncIndex lastConfigIndex; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index baff19e3d4..a120ecf9db 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -263,7 +263,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader void vnodeSnapReaderClose(SVSnapReader *pReader); int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData); // SVSnapWriter -int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter); +int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter); int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index c56644ca2c..cf04471be0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -49,6 +49,26 @@ struct SVSnapReader { SRSmaSnapReader *pRsmaReader; }; +static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) { + int32_t code = -1; + STsdbSnapPartList *pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + goto _out; + } + code = 0; +_out: + tsdbSnapPartListDestroy(&pList); + return code; +} + int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader) { int32_t code = 0; int64_t sver = pParam->start; @@ -64,6 +84,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->sver = sver; pReader->ever = ever; + // snapshot info if (pParam->data) { SSyncTLV *datHead = (void *)pParam->data; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { @@ -72,37 +93,29 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader } int32_t offset = 0; + TSnapRangeArray **ppRanges = NULL; + while (offset + sizeof(SSyncTLV) < datHead->len) { - SSyncTLV *sectHead = (void *)(datHead->val + offset); - offset += sizeof(SSyncTLV) + sectHead->len; - void *buf = sectHead->val; - int32_t bufLen = sectHead->len; - ASSERT(sectHead->typ == SNAP_DATA_TSDB || sectHead->typ == SNAP_DATA_RSMA1); - STsdbSnapPartList *pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + SSyncTLV *subField = (void *)(datHead->val + offset); + offset += sizeof(SSyncTLV) + subField->len; + void *buf = subField->val; + int32_t bufLen = subField->len; + switch (subField->typ) { + case SNAP_DATA_TSDB: + ppRanges = &pReader->pRanges; + break; + case SNAP_DATA_RSMA1: + ppRanges = &pReader->pRsmaRanges; + break; + default: + vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), + subField->typ); + goto _err; } - if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { - terrno = TSDB_CODE_INVALID_DATA_FMT; - goto _err; - } - TSnapRangeArray **ppRanges = NULL; - if (sectHead->typ == SNAP_DATA_TSDB) { - ppRanges = &pReader->pRanges; - } else if (sectHead->typ == SNAP_DATA_RSMA1) { - ppRanges = &pReader->pRsmaRanges; - } - if (ppRanges == NULL) { - tsdbSnapPartListDestroy(&pList); - continue; - } - if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - tsdbSnapPartListDestroy(&pList); goto _err; } - tsdbSnapPartListDestroy(&pList); } } @@ -414,6 +427,7 @@ struct SVSnapWriter { // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb + TSnapRangeArray *pRanges; STsdbSnapWriter *pTsdbSnapWriter; // tq STqSnapWriter *pTqSnapWriter; @@ -423,12 +437,15 @@ struct SVSnapWriter { SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; // rsma + TSnapRangeArray *pRsmaRanges; SRSmaSnapWriter *pRsmaSnapWriter; }; -int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) { +int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t code = 0; SVSnapWriter *pWriter = NULL; + int64_t sver = pParam->start; + int64_t ever = pParam->end; // commit memory data vnodeAsyncCommit(pVnode); @@ -447,6 +464,41 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr // inc commit ID pWriter->commitID = ++pVnode->state.commitID; + // snapshot info + if (pParam->data) { + SSyncTLV *datHead = (void *)pParam->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _err; + } + + int32_t offset = 0; + TSnapRangeArray **ppRanges = NULL; + + while (offset + sizeof(SSyncTLV) < datHead->len) { + SSyncTLV *subField = (void *)(datHead->val + offset); + offset += sizeof(SSyncTLV) + subField->len; + void *buf = subField->val; + int32_t bufLen = subField->len; + switch (subField->typ) { + case SNAP_DATA_TSDB: + ppRanges = &pWriter->pRanges; + break; + case SNAP_DATA_RSMA1: + ppRanges = &pWriter->pRsmaRanges; + break; + default: + vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), + subField->typ); + goto _err; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _err; + } + } + } + vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode), sver, ever, pWriter->commitID); *ppWriter = pWriter; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index e676219b11..b73c9b8c65 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -491,8 +491,7 @@ static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **p } static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) { - SVnode *pVnode = pFsm->data; - SSnapshotParam *pSnapshotParam = pParam; + SVnode *pVnode = pFsm->data; do { int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); @@ -505,7 +504,7 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void } } while (true); - int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter); + int32_t code = vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter); return code; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 681d256ec9..99e8fd55a2 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -368,6 +368,17 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { pReceiver->pWriter = NULL; } + // free data of snapshot info + if (pReceiver->snapshotParam.data) { + taosMemoryFree(pReceiver->snapshotParam.data); + pReceiver->snapshotParam.data = NULL; + } + + if (pReceiver->snapshot.data) { + taosMemoryFree(pReceiver->snapshot.data); + pReceiver->snapshot.data = NULL; + } + // free receiver taosMemoryFree(pReceiver); } @@ -652,6 +663,17 @@ _SEND_REPLY: if (snapInfo.data) { pRspMsg->payloadType = snapInfo.typ; memcpy(pRspMsg->data, snapInfo.data, dataLen); + + // save snapshot info + SSnapshotParam *pParam = &pReceiver->snapshotParam; + void *data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pParam->data = data; + memcpy(pParam->data, snapInfo.data, dataLen); } // send msg @@ -941,7 +963,7 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend terrno = TSDB_CODE_INVALID_DATA_FMT; return -1; } - pSender->snapshotParam.data = pMsg->data; + pSender->snapshotParam.data = (void *)pMsg->data; } int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); From 0c41fa56dc113ed5294eb2d27b1fc4a332631710 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 18 Sep 2023 14:46:26 +0800 Subject: [PATCH 109/177] feat: filter pExclude in tsdbFSCreateCopyRangedSnapshot --- source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/sma/smaSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 46 ++++++++++++++- source/dnode/vnode/src/tsdb/tsdbFS2.h | 7 ++- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 68 ++++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbFSet2.h | 3 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 19 +++++- 8 files changed, 144 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 27a393abf4..d5e1585af4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -301,7 +301,7 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader); int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData); // STsdbSnapWriter ======================================== -int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter); +int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRanges, STsdbSnapWriter** ppWriter); int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr); int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); @@ -358,7 +358,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader); int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== -int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter); +int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void* pRanges, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 2a010f5a84..ca46b4728f 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -128,7 +128,7 @@ struct SRSmaSnapWriter { STsdbSnapWriter* pDataWriter[TSDB_RETENTION_L2]; }; -int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter) { +int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void* pRanges, SRSmaSnapWriter** ppWriter) { int32_t code = 0; int32_t lino = 0; SVnode* pVnode = pSma->pVnode; @@ -147,7 +147,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, &pWriter->pDataWriter[i]); + code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, pRanges, &pWriter->pDataWriter[i]); TSDB_CHECK_CODE(code, lino, _exit); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index d71473b079..759fded522 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -983,6 +983,50 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, + TFileOpArray *fopArr) { + int32_t code = 0; + STFileSet *fset; + STFileSet *fset1; + + fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray)); + if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + TARRAY2_INIT(fsetArr[0]); + + int32_t i = 0; + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + TARRAY2_FOREACH(fs->fSetArr, fset) { + int64_t ever = VERSION_MAX; + while (pExclude && i < TARRAY2_SIZE(pExclude)) { + STSnapRange *u = TARRAY2_GET(pExclude, i); + if (fset->fid > u->fid) { + i++; + continue; + } + if (fset->fid == u->fid) { + ever = u->sver - 1; + i++; + } + } + + code = tsdbTFileSetFilteredInitDup(fs->tsdb, fset, ever, &fset1, fopArr); + if (code) break; + + code = TARRAY2_APPEND(fsetArr[0], fset1); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFree(fsetArr[0]); + fsetArr[0] = NULL; + } + return code; +} + int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, TSnapRangeArray **fsrArr) { int32_t code = -1; @@ -1009,12 +1053,12 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev i++; continue; } - if (fset->fid == u->fid) { sver1 = u->sver; i++; } } + tsdbInfo("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index f8b87b8a84..73b27a8fbd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -52,9 +52,12 @@ int32_t tsdbCloseFS(STFileSystem **fs); int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); -int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsrArr); +int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pEx, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, + TFileOpArray *fopArr); +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr); +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, TSnapRangeArray **fsrArr); int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr); // txn diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index e0434b7da6..dd86d01598 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -65,6 +65,34 @@ static int32_t tsdbSttLvlInitRef(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lv return 0; } +static int32_t tsdbSttLvlFilteredInitEx(STsdb *pTsdb, const SSttLvl *lvl1, int64_t ever, SSttLvl **lvl, + TFileOpArray *fopArr) { + int32_t code = tsdbSttLvlInit(lvl1->level, lvl); + if (code) return code; + + const STFileObj *fobj1; + TARRAY2_FOREACH(lvl1->fobjArr, fobj1) { + if (fobj1->f->maxVer <= ever) { + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + } else { + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = fobj1->f->fid, + .of = fobj1->f[0], + }; + TARRAY2_APPEND(fopArr, op); + } + } + return 0; +} + static void tsdbSttLvlRemoveFObj(void *data) { tsdbTFileObjRemove(*(STFileObj **)data); } static void tsdbSttLvlRemove(SSttLvl **lvl) { TARRAY2_DESTROY(lvl[0]->fobjArr, tsdbSttLvlRemoveFObj); @@ -458,6 +486,46 @@ int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } +int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, + TFileOpArray *fopArr) { + int32_t code = tsdbTFileSetInit(fset1->fid, fset); + if (code) return code; + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset1->farr[ftype] == NULL) continue; + STFileObj *fobj = fset1->farr[ftype]; + if (fobj->f->maxVer <= ever) { + code = tsdbTFileObjInit(pTsdb, fobj->f, &fset[0]->farr[ftype]); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + } else { + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + TARRAY2_APPEND(fopArr, op); + } + } + + const SSttLvl *lvl1; + TARRAY2_FOREACH(fset1->lvlArr, lvl1) { + SSttLvl *lvl; + code = tsdbSttLvlFilteredInitEx(pTsdb, lvl1, ever, &lvl, fopArr); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + code = TARRAY2_APPEND(fset[0]->lvlArr, lvl); + if (code) return code; + } + + return 0; +} + int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index e6d78a8cfe..c78cc179df 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -46,6 +46,9 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs int32_t tsdbTFileSetClear(STFileSet **fset); int32_t tsdbTFileSetRemove(STFileSet **fset); +int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, + TFileOpArray *fopArr); + int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr); int32_t tsdbTSnapRangeClear(STSnapRange **fsr); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 9710eee6c4..3b74475870 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1028,7 +1028,7 @@ _exit: return code; } -int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** writer) { +int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRanges, STsdbSnapWriter** writer) { int32_t code = 0; int32_t lino = 0; @@ -1052,7 +1052,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr writer[0]->compactVersion = INT64_MAX; writer[0]->now = taosGetTimestampMs(); - code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr); + code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index cf04471be0..b2fbdc07e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -158,6 +158,11 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { if (pReader->pRanges) { tsdbSnapRangeArrayDestroy(&pReader->pRanges); } + + if (pReader->pRsmaRanges) { + tsdbSnapRangeArrayDestroy(&pReader->pRsmaRanges); + } + taosMemoryFree(pReader); } @@ -543,6 +548,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } + if (pWriter->pRanges) { + tsdbSnapRangeArrayDestroy(&pWriter->pRanges); + } + if (pWriter->pTsdbSnapWriter) { code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback); if (code) goto _exit; @@ -577,6 +586,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } + if (pWriter->pRsmaRanges) { + tsdbSnapRangeArrayDestroy(&pWriter->pRsmaRanges); + } + if (pWriter->pRsmaSnapWriter) { code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback); if (code) goto _exit; @@ -663,7 +676,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { case SNAP_DATA_DEL: { // tsdb if (pWriter->pTsdbSnapWriter == NULL) { - code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); + code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges, + &pWriter->pTsdbSnapWriter); if (code) goto _err; } @@ -723,7 +737,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { case SNAP_DATA_QTASK: { // rsma1/rsma2/qtask for rsma if (pWriter->pRsmaSnapWriter == NULL) { - code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, &pWriter->pRsmaSnapWriter); + code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, pWriter->pRsmaRanges, + &pWriter->pRsmaSnapWriter); if (code) goto _err; } From eb4e2aa58fbae2a33ee6fe2a8fe02d3d9ba880ac Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 20 Sep 2023 09:54:28 +0800 Subject: [PATCH 110/177] feat: restore incomplete fsm state with maxVerValid via snapshot replication --- include/common/tcommon.h | 4 +- include/libs/sync/sync.h | 13 +- include/util/taoserror.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 5 + source/dnode/vnode/src/tsdb/tsdbFS2.c | 33 ++-- source/dnode/vnode/src/tsdb/tsdbFS2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 1 + source/dnode/vnode/src/tsdb/tsdbFSet2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 65 ++++++-- source/dnode/vnode/src/vnd/vnodeOpen.c | 9 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 +- source/dnode/vnode/src/vnd/vnodeSync.c | 13 ++ source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncMessage.h | 2 +- source/libs/sync/inc/syncSnapshot.h | 6 +- source/libs/sync/src/syncAppendEntries.c | 9 +- source/libs/sync/src/syncElection.c | 5 + source/libs/sync/src/syncMain.c | 16 +- source/libs/sync/src/syncPipeline.c | 15 +- source/libs/sync/src/syncSnapshot.c | 175 ++++++++++++--------- source/util/src/terror.c | 1 + 21 files changed, 242 insertions(+), 141 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 6f4f15d1e8..72aab9adf0 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -55,8 +55,8 @@ typedef struct SSessionKey { } SSessionKey; typedef struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; + int64_t minVer; + int64_t maxVer; } SVersionRange; static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) { diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 50e60d2ef4..71c56e8c86 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -36,8 +36,7 @@ extern "C" { #define SYNC_DEL_WAL_MS (1000 * 60) #define SYNC_ADD_QUORUM_COUNT 3 #define SYNC_VNODE_LOG_RETENTION (TSDB_SYNC_LOG_BUFFER_RETENTION + 1) -#define SNAPSHOT_MAX_CLOCK_SKEW_MS 1000 * 10 -#define SNAPSHOT_WAIT_MS 1000 * 30 +#define SNAPSHOT_WAIT_MS 1000 * 5 #define SYNC_MAX_RETRY_BACKOFF 5 #define SYNC_LOG_REPL_RETRY_WAIT_MS 100 @@ -88,10 +87,9 @@ typedef enum { } ESyncRole; typedef enum { - TAOS_SYNC_SNAP_INFO_BRIEF = 0, - TAOS_SYNC_SNAP_INFO_FULL = 1, - TAOS_SYNC_SNAP_INFO_DIFF = 2, -} ESyncSnapInfoTyp; + SYNC_FSM_STATE_NORMAL = 0, + SYNC_FSM_STATE_INCOMPLETE, +} ESyncFsmState; typedef struct SNodeInfo { int64_t clusterId; @@ -155,8 +153,9 @@ typedef struct SSnapshotParam { } SSnapshotParam; typedef struct SSnapshot { - int32_t typ; + int32_t type; SSyncTLV* data; + ESyncFsmState state; SyncIndex lastApplyIndex; SyncTerm lastApplyTerm; SyncIndex lastConfigIndex; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 39ae3fb97a..6fbe4422ac 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -557,7 +557,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) // 2.x // #define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) // 2.x // #define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) // 2.x -// #define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) // 2.x +#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) // #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x // #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x // #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4cea7c5e85..1d17d616ab 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -1025,6 +1025,11 @@ struct STsdbFilterInfo { TABLEID tbid; }; +enum { + TSDB_FS_STATE_NORMAL = 0, + TSDB_FS_STATE_INCOMPLETE, +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 759fded522..ef2f81fa02 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -38,13 +38,6 @@ typedef struct { STFileHashEntry **buckets; } STFileHash; -enum { - TSDB_FS_STATE_NONE = 0, - TSDB_FS_STATE_OPEN, - TSDB_FS_STATE_EDIT, - TSDB_FS_STATE_CLOSE, -}; - static const char *gCurrentFname[] = { [TSDB_FCURRENT] = "current.json", [TSDB_FCURRENT_C] = "current.c.json", @@ -57,7 +50,7 @@ static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { fs[0]->tsdb = pTsdb; tsem_init(&fs[0]->canEdit, 0, 1); - fs[0]->state = TSDB_FS_STATE_NONE; + fs[0]->fsstate = TSDB_FS_STATE_NORMAL; fs[0]->neid = 0; TARRAY2_INIT(fs[0]->fSetArr); TARRAY2_INIT(fs[0]->fSetArrTmp); @@ -496,6 +489,7 @@ static void tsdbFSDestroyFileObjHash(STFileHash *hash) { static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { int32_t code = 0; int32_t lino = 0; + int32_t corrupt = false; { // scan each file STFileSet *fset = NULL; @@ -503,8 +497,12 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { // data file for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { if (fset->farr[ftype] == NULL) continue; - code = tsdbFSDoScanAndFixFile(fs, fset->farr[ftype]); - TSDB_CHECK_CODE(code, lino, _exit); + STFileObj *fobj = fset->farr[ftype]; + code = tsdbFSDoScanAndFixFile(fs, fobj); + if (code) { + fset->maxVerValid = TMIN(fset->maxVerValid, fobj->f->minVer - 1); + corrupt = true; + } } // stt file @@ -513,12 +511,21 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { STFileObj *fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { code = tsdbFSDoScanAndFixFile(fs, fobj); - TSDB_CHECK_CODE(code, lino, _exit); + if (code) { + fset->maxVerValid = TMIN(fset->maxVerValid, fobj->f->minVer - 1); + corrupt = true; + } } } } } + if (corrupt) { + tsdbError("vgId:%d, not to clear unreferenced files since some fset corrupted", TD_VID(fs->tsdb->pVnode)); + fs->fsstate = TSDB_FS_STATE_INCOMPLETE; + goto _exit; + } + { // clear unreferenced files STfsDir *dir = tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path); if (dir == NULL) { @@ -1009,6 +1016,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclu ever = u->sver - 1; i++; } + break; } code = tsdbTFileSetFilteredInitDup(fs->tsdb, fset, ever, &fset1, fopArr); @@ -1057,8 +1065,11 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev sver1 = u->sver; i++; } + break; } + if (sver1 > ever1) continue; + tsdbInfo("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 73b27a8fbd..851459df53 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -101,7 +101,7 @@ struct STFSBgTask { struct STFileSystem { STsdb *tsdb; tsem_t canEdit; - int32_t state; + int32_t fsstate; int64_t neid; EFEditT etype; TFileSetArray fSetArr[1]; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index dd86d01598..620fcb3a47 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -452,6 +452,7 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { if (fset[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; fset[0]->fid = fid; + fset[0]->maxVerValid = VERSION_MAX; TARRAY2_INIT(fset[0]->lvlArr); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index c78cc179df..ea0f99f68e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -84,7 +84,7 @@ struct SSttLvl { struct STFileSet { int32_t fid; - int8_t stat; + int64_t maxVerValid; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array }; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 3b74475870..6ee7112906 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -436,8 +436,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, sver, ever, - type); + tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), + sver, ever, type); } return code; } @@ -1103,6 +1103,8 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { TSDB_CHECK_CODE(code, lino, _exit); } + writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL; + taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); } tsdbFSEnableBgTask(tsdb->pFS); @@ -1218,14 +1220,28 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP goto _err; } + p->fid = fset->fid; + + int32_t code = 0; int32_t typ = 0; + int32_t corrupt = false; + int32_t count = 0; for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset->farr[ftype] == NULL) continue; typ = tsdbFTypeToSRangeTyp(ftype); ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); STFile* f = fset->farr[ftype]->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); + continue; + } + count++; SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); } typ = TSDB_SNAP_RANGE_TYP_STT; @@ -1234,10 +1250,24 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP STFileObj* fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { STFile* f = fobj->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); + continue; + } + count++; SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); } } + if (corrupt && count == 0) { + SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } ppSP[0] = p; return 0; @@ -1272,7 +1302,8 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { break; } ASSERT(pItem != NULL); - TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); + ASSERT(code == 0); } taosThreadRwlockUnlock(&fs->tsdb->rwLock); @@ -1432,18 +1463,22 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - int64_t ever = -1; + int64_t maxVerValid = -1; int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; for (int32_t i = 0; i < typMax; i++) { SVerRangeList* iList = &part->verRanges[i]; - SVersionRange r = {0}; - TARRAY2_FOREACH(iList, r) { - if (r.maxVer < r.minVer) continue; - ever = TMAX(ever, r.maxVer); + SVersionRange vr = {0}; + TARRAY2_FOREACH(iList, vr) { + if (vr.maxVer < vr.minVer) { + continue; + } + maxVerValid = TMAX(maxVerValid, vr.maxVer); } } - r->sver = ever + 1; + r->fid = part->fid; + r->sver = maxVerValid + 1; r->ever = VERSION_MAX; + tsdbInfo("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); TARRAY2_APPEND(pDiff, r); } ppRanges[0] = pDiff; @@ -1473,14 +1508,16 @@ void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { } int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { - if (pSnap->typ != TDMT_SYNC_PREP_SNAPSHOT && pSnap->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + pSnap->state = pTsdb->pFS->fsstate; + if (pSnap->type != TDMT_SYNC_PREP_SNAPSHOT && pSnap->type != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { return 0; } + int code = -1; STsdbSnapPartList* pList = tsdbGetSnapPartList(pTsdb->pFS); if (pList == NULL) goto _out; - if (pSnap->typ == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { } void* buf = NULL; @@ -1499,7 +1536,7 @@ int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { // header SSyncTLV* datHead = (void*)pSnap->data; - datHead->typ = pSnap->typ; + datHead->typ = pSnap->type; datHead->len = 0; // tsdb diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index fada83a7f1..5084cc2ff5 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include "sync.h" +#include "tsdb.h" #include "vnd.h" #include "vndCos.h" @@ -517,10 +519,3 @@ ESyncRole vnodeGetRole(SVnode *pVnode) { return syncGetRole(pVnode->sync); } void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } - -int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { - pSnap->lastApplyIndex = pVnode->state.committed; - pSnap->lastApplyTerm = pVnode->state.commitTerm; - pSnap->lastConfigIndex = -1; - return tsdbSnapGetInfo(pVnode->pTsdb, pSnap); -} diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index b2fbdc07e9..0874e5e0d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -537,7 +537,9 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * char dir[TSDB_FILENAME_LEN] = {0}; vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); - vnodeCommitInfo(dir); + code = vnodeCommitInfo(dir); + if (code) goto _exit; + } else { vnodeRollback(pWriter->pVnode); } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b73c9b8c65..ba142ddb6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -15,6 +15,8 @@ #define _DEFAULT_SOURCE #include "tq.h" +#include "sync.h" +#include "tsdb.h" #include "vnd.h" #define BATCH_ENABLE 0 @@ -783,3 +785,14 @@ bool vnodeIsLeader(SVnode *pVnode) { return true; } + +int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { + pSnap->lastApplyIndex = pVnode->state.committed; + pSnap->lastApplyTerm = pVnode->state.commitTerm; + pSnap->lastConfigIndex = -1; + + int32_t code = tsdbSnapGetInfo(pVnode->pTsdb, pSnap); + + pSnap->state = (pSnap->state == TSDB_FS_STATE_INCOMPLETE) ? SYNC_FSM_STATE_INCOMPLETE : SYNC_FSM_STATE_NORMAL; + return code; +} diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 870cdd6a72..cec1a12024 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -139,6 +139,7 @@ typedef struct SSyncNode { SSyncFSM* pFsm; int32_t quorum; SRaftId leaderCache; + ESyncFsmState fsmState; // life cycle int64_t rid; diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index c0d3663a8f..9054f47d37 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -116,7 +116,7 @@ typedef struct SyncAppendEntriesReply { SyncIndex matchIndex; SyncIndex lastSendIndex; int64_t startTime; - int16_t reserved; + int16_t fsmState; } SyncAppendEntriesReply; typedef struct SyncHeartbeat { diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 063b4f51f5..2a19945c5a 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -31,7 +31,7 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 typedef struct SSyncSnapshotSender { - bool start; + int8_t start; int32_t seq; int32_t ack; void *pReader; @@ -60,8 +60,8 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finis int32_t snapshotReSend(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { - // update when pre snapshot - bool start; + // update when prep snapshot + int8_t start; int32_t ack; SyncTerm term; SRaftId fromId; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 925988f43a..8ae1dd2a54 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -155,6 +155,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pEntry->term); + if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) { + pReply->fsmState = ths->fsmState; + sError("vgId:%d, not allow to accept sync log msg due to incomplete fsm state", ths->vgId); + syncEntryDestroy(pEntry); + goto _SEND_RESPONSE; + } + // accept if (syncLogBufferAccept(ths->pLogBuf, ths, pEntry, pMsg->prevLogTerm) < 0) { goto _SEND_RESPONSE; @@ -175,7 +182,7 @@ _SEND_RESPONSE: (void)syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp); // commit index, i.e. leader notice me - if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { + if (ths->fsmState != SYNC_FSM_STATE_INCOMPLETE && syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit raft fsm log since %s.", ths->vgId, terrstr()); } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 86e28db90c..b4e2049a64 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -71,6 +71,11 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { } int32_t syncNodeElect(SSyncNode* pSyncNode) { + if (pSyncNode->fsmState == SYNC_FSM_STATE_INCOMPLETE) { + sNError(pSyncNode, "ignore leader hb timeout due to incomplete fsm state"); + return -1; + } + sNInfo(pSyncNode, "begin election"); pSyncNode->electNum++; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index eca499cf28..8ddd55d906 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1009,6 +1009,10 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { commitIndex = snapshot.lastApplyIndex; sNTrace(pSyncNode, "reset commit index by snapshot"); } + pSyncNode->fsmState = snapshot.state; + if (pSyncNode->fsmState) { + sError("vgId:%d, fsm state incomplete.", pSyncNode->vgId); + } } pSyncNode->commitIndex = commitIndex; sInfo("vgId:%d, sync node commitIndex initialized as %" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); @@ -1163,7 +1167,8 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { pSyncNode->commitIndex = TMAX(pSyncNode->commitIndex, commitIndex); sInfo("vgId:%d, restore sync until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); - if (syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, pSyncNode->commitIndex) < 0) { + if (pSyncNode->fsmState != SYNC_FSM_STATE_INCOMPLETE && + syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, pSyncNode->commitIndex) < 0) { return -1; } @@ -1455,10 +1460,9 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pNode, SRpcMsg } if (code < 0) { - sError("vgId:%d, sync send msg by id error, epset:%p dnode:%d addr:%" PRId64 " err:0x%x", pNode->vgId, epSet, - DID(destRaftId), destRaftId->addr, terrno); + sError("vgId:%d, failed to send sync msg since %s. epset:%p dnode:%d addr:%" PRId64, pNode->vgId, terrstr(), epSet, + DID(destRaftId), destRaftId->addr); rpcFreeCont(pMsg->pCont); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } return code; @@ -2895,7 +2899,7 @@ _out:; // single replica (void)syncNodeUpdateCommitIndex(ths, matchIndex); - if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { + if (ths->fsmState != SYNC_FSM_STATE_INCOMPLETE && syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit until commitIndex:%" PRId64 "", ths->vgId, ths->commitIndex); code = -1; } @@ -3139,7 +3143,7 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (pMsg->currentTerm == matchTerm) { (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex); } - if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { + if (ths->fsmState != SYNC_FSM_STATE_INCOMPLETE && syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit raft log since %s. commit index:%" PRId64 "", ths->vgId, terrstr(), ths->commitIndex); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 019f8f7e62..a38d67a388 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -839,14 +839,16 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn return 0; } - if (pMsg->success == false && pMsg->matchIndex >= pMsg->lastSendIndex) { - sWarn("vgId:%d, failed to rollback match index. peer: dnode:%d, match index:%" PRId64 ", last sent:%" PRId64, - pNode->vgId, DID(&destId), pMsg->matchIndex, pMsg->lastSendIndex); + if (pMsg->fsmState == SYNC_FSM_STATE_INCOMPLETE || (!pMsg->success && pMsg->matchIndex >= pMsg->lastSendIndex)) { + char* msg1 = "rollback match index failure"; + char* msg2 = "incomplete fsm state"; + sInfo("vgId:%d, snapshot replication to dnode:%d. reason:%s, match index:%" PRId64 ", last sent:%" PRId64, + pNode->vgId, DID(&destId), (pMsg->fsmState == SYNC_FSM_STATE_INCOMPLETE ? msg2 : msg1), pMsg->matchIndex, + pMsg->lastSendIndex); if (syncNodeStartSnapshot(pNode, &destId) < 0) { sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId)); return -1; } - sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId)); return 0; } } @@ -1000,10 +1002,9 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { pMgr->endIndex = index + 1; if (barrier) { - sInfo("vgId:%d, replicated sync barrier to dest:%" PRIx64 ". index:%" PRId64 ", term:%" PRId64 + sInfo("vgId:%d, replicated sync barrier to dnode:%d. index:%" PRId64 ", term:%" PRId64 ", repl mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, pDestId->addr, index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, - pMgr->endIndex); + pNode->vgId, DID(pDestId), index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); break; } } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 99e8fd55a2..383fda89b0 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -44,8 +44,8 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; pSender->term = raftStoreGetTerm(pSyncNode); - pSender->startTime = 0; - pSender->endTime = 0; + pSender->startTime = -1; + pSender->endTime = -1; pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); pSender->finish = false; @@ -71,11 +71,16 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { taosMemoryFree(pSender); } -bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } +bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return atomic_load_8(&pSender->start); } int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { int32_t code = -1; - pSender->start = true; + + int8_t started = atomic_val_compare_exchange_8(&pSender->start, false, true); + if (started) return 0; + + taosMsleep(1); + pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; pSender->pReader = NULL; @@ -92,13 +97,13 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { memset(&pSender->lastConfig, 0, sizeof(pSender->lastConfig)); pSender->sendingMS = 0; pSender->term = raftStoreGetTerm(pSender->pSyncNode); - pSender->startTime = taosGetTimestampMs(); + pSender->startTime = taosGetMonoTimestampMs(); pSender->lastSendTime = pSender->startTime; pSender->finish = false; // Get full snapshot info SSyncNode *pSyncNode = pSender->pSyncNode; - SSnapshot snapInfo = {.typ = TDMT_SYNC_PREP_SNAPSHOT}; + SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT}; if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) { sSError(pSender, "snapshot get info failure since %s", terrstr()); goto _out; @@ -130,11 +135,11 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; - pMsg->startTime = pSender->startTime; + pMsg->startTime = atomic_load_64(&pSender->startTime); pMsg->seq = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; if (dataLen > 0) { - pMsg->payloadType = snapInfo.typ; + pMsg->payloadType = snapInfo.type; memcpy(pMsg->data, snapInfo.data, dataLen); } @@ -160,7 +165,9 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { sSDebug(pSender, "snapshot sender stop, finish:%d reader:%p", finish, pSender->pReader); // update flag - pSender->start = false; + int8_t stopped = !atomic_val_compare_exchange_8(&pSender->start, true, false); + if (stopped) return; + pSender->finish = finish; pSender->endTime = taosGetTimestampMs(); @@ -223,6 +230,7 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; + pMsg->startTime = pSender->startTime; pMsg->seq = pSender->seq; if (pSender->pCurrentBlock != NULL) { @@ -286,7 +294,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { static int32_t snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { if (pMsg->ack != pSender->seq) { sSError(pSender, "snapshot sender update seq failed, ack:%d seq:%d", pMsg->ack, pSender->seq); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } @@ -301,8 +309,6 @@ static int32_t snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSn // return 1, last snapshot finish ok // return -1, error int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { - sNInfo(pSyncNode, "snapshot sender starting ..."); - SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId); if (pSender == NULL) { sNError(pSyncNode, "snapshot sender start error since get failed"); @@ -310,12 +316,12 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { } if (snapshotSenderIsStart(pSender)) { - sSInfo(pSender, "snapshot sender already start, ignore"); + sSDebug(pSender, "snapshot sender already start, ignore"); return 0; } if (pSender->finish && taosGetTimestampMs() - pSender->endTime < SNAPSHOT_WAIT_MS) { - sSInfo(pSender, "snapshot sender start too frequently, ignore"); + sSDebug(pSender, "snapshot sender start too frequently, ignore"); return 0; } @@ -342,6 +348,7 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from } pReceiver->start = false; + pReceiver->startTime = 0; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; pReceiver->pWriter = NULL; pReceiver->pSyncNode = pSyncNode; @@ -384,7 +391,7 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { } bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { - return (pReceiver != NULL ? pReceiver->start : false); + return (pReceiver != NULL ? atomic_load_8(&pReceiver->start) : false); } static int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { @@ -423,11 +430,14 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p return; } - pReceiver->start = true; + int8_t started = atomic_val_compare_exchange_8(&pReceiver->start, false, true); + if (started) return; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; pReceiver->term = raftStoreGetTerm(pReceiver->pSyncNode); pReceiver->fromId = pPreMsg->srcId; pReceiver->startTime = pPreMsg->startTime; + ASSERT(pReceiver->startTime); // event log sRInfo(pReceiver, "snapshot receiver is start"); @@ -438,6 +448,9 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { sRInfo(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); + int8_t stopped = !atomic_val_compare_exchange_8(&pReceiver->start, true, false); + if (stopped) return; + if (pReceiver->pWriter != NULL) { int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, &pReceiver->snapshot); @@ -448,8 +461,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } else { sRInfo(pReceiver, "snapshot receiver stop, writer is null"); } - - pReceiver->start = false; } // when recv last snapshot block, apply data into snapshot @@ -499,6 +510,10 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap // update progress pReceiver->ack = SYNC_SNAPSHOT_SEQ_END; + SSnapshot snapshot = {0}; + pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); + pReceiver->pSyncNode->fsmState = snapshot.state; + } else { sRError(pReceiver, "snapshot receiver finish error since writer is null"); return -1; @@ -582,7 +597,7 @@ static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pM // ignore sRError(pReceiver, "snapshot receiver startTime:%" PRId64 " < msg startTime:%" PRId64 " ignore", pReceiver->startTime, pMsg->startTime); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; code = terrno; goto _SEND_REPLY; } @@ -593,33 +608,18 @@ static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pM } _START_RECEIVER: - if (timeNow - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) { - sRError(pReceiver, "snapshot receiver time skew too much, now:%" PRId64 " msg startTime:%" PRId64, timeNow, - pMsg->startTime); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - code = terrno; - } else { - // waiting for clock match - while (timeNow < pMsg->startTime) { - sRInfo(pReceiver, "snapshot receiver pre waitting for true time, now:%" PRId64 ", startTime:%" PRId64, timeNow, - pMsg->startTime); - taosMsleep(10); - timeNow = taosGetTimestampMs(); - } - - if (snapshotReceiverIsStart(pReceiver)) { - sRInfo(pReceiver, "snapshot receiver already start and force stop pre one"); - snapshotReceiverStop(pReceiver); - } - - snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender + if (snapshotReceiverIsStart(pReceiver)) { + sRInfo(pReceiver, "snapshot receiver already start and force stop pre one"); + snapshotReceiverStop(pReceiver); } + snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender + _SEND_REPLY: // build msg ; // make complier happy - SSnapshot snapInfo = {.typ = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; + SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; if (pMsg->dataLen > 0) { void *data = taosMemoryCalloc(1, pMsg->dataLen); @@ -655,13 +655,15 @@ _SEND_REPLY: pRspMsg->term = raftStoreGetTerm(pSyncNode); pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pReceiver->startTime; + pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pMsg->seq; // receiver maybe already closed pRspMsg->code = code; pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); + ASSERT(pRspMsg->startTime); + if (snapInfo.data) { - pRspMsg->payloadType = snapInfo.typ; + pRspMsg->payloadType = snapInfo.type; memcpy(pRspMsg->data, snapInfo.data, dataLen); // save snapshot info @@ -704,6 +706,7 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p if (pReceiver->startTime != pMsg->startTime) { sRError(pReceiver, "snapshot receiver begin failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, pReceiver->startTime, pMsg->startTime); + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; goto _SEND_REPLY; } @@ -732,11 +735,13 @@ _SEND_REPLY: pRspMsg->term = raftStoreGetTerm(pSyncNode); pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pReceiver->startTime; + pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; + ASSERT(pRspMsg->startTime); + // send msg syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver begin"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { @@ -751,17 +756,17 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend // condition 4 // transfering SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; - - // waiting for clock match int64_t timeNow = taosGetTimestampMs(); - while (timeNow < pMsg->startTime) { - sRInfo(pReceiver, "snapshot receiver receiving waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow, - pMsg->startTime); - taosMsleep(10); - timeNow = taosGetTimestampMs(); + int32_t code = 0; + + if (pReceiver->startTime != pMsg->startTime) { + sRError(pReceiver, "snapshot receive failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, + pReceiver->startTime, pMsg->startTime); + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + code = terrno; + goto _SEND_REPLY; } - int32_t code = 0; if (snapshotReceiverGotData(pReceiver, pMsg) != 0) { code = terrno; if (code >= SYNC_SNAPSHOT_SEQ_INVALID) { @@ -769,6 +774,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend } } +_SEND_REPLY: // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { @@ -782,11 +788,12 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend pRspMsg->term = raftStoreGetTerm(pSyncNode); pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pReceiver->startTime; + pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; + ASSERT(pRspMsg->startTime); // send msg syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver received"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { @@ -801,21 +808,23 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs // condition 2 // end, finish FSM SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; - - // waiting for clock match int64_t timeNow = taosGetTimestampMs(); - while (timeNow < pMsg->startTime) { - sRInfo(pReceiver, "snapshot receiver finish waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow, - pMsg->startTime); - taosMsleep(10); - timeNow = taosGetTimestampMs(); + int32_t code = 0; + + if (pReceiver->startTime != pMsg->startTime) { + sRError(pReceiver, "snapshot end failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, + pReceiver->startTime, pMsg->startTime); + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + code = terrno; + goto _SEND_REPLY; } - int32_t code = snapshotReceiverFinish(pReceiver, pMsg); + code = snapshotReceiverFinish(pReceiver, pMsg); if (code == 0) { snapshotReceiverStop(pReceiver); } +_SEND_REPLY: // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { @@ -829,7 +838,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs pRspMsg->term = raftStoreGetTerm(pSyncNode); pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pReceiver->startTime; + pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; @@ -945,13 +954,6 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend sSInfo(pSender, "prepare snapshot, recv-begin:%" PRId64 ", snapshot.last:%" PRId64 ", snapshot.term:%" PRId64, pMsg->snapBeginIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm); - if (pMsg->snapBeginIndex > snapshot.lastApplyIndex) { - sSError(pSender, "prepare snapshot failed since beginIndex:%" PRId64 " larger than applyIndex:%" PRId64, - pMsg->snapBeginIndex, snapshot.lastApplyIndex); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - // update sender pSender->snapshot = snapshot; @@ -964,6 +966,7 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend return -1; } pSender->snapshotParam.data = (void *)pMsg->data; + sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); } int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); @@ -997,6 +1000,11 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend pSendMsg->startTime = pSender->startTime; pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + ASSERT(pSendMsg->startTime); + + sSInfo(pSender, "begin snapshot replication to dnode %d. startTime:%" PRId64, DID(&pSendMsg->destId), + pSendMsg->startTime); + // send msg syncLogSendSyncSnapshotSend(pSyncNode, pSendMsg, "snapshot sender reply pre"); if (syncNodeSendMsgById(&pSendMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) { @@ -1019,7 +1027,7 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "maybe replica already dropped"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } @@ -1031,6 +1039,25 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } + if (!snapshotSenderIsStart(pSender)) { + sSError(pSender, "snapshot sender not started yet. sender startTime:%" PRId64 ", msg startTime:%" PRId64, + pSender->startTime, pMsg->startTime); + return -1; + } + + if (pMsg->startTime < pSender->startTime) { + sSError(pSender, "ignore stale rsp received. sender startTime:%" PRId64 ", msg startTime:%" PRId64, + pSender->startTime, pMsg->startTime); + terrno = pMsg->code; + return -1; + } else if (pMsg->startTime > pSender->startTime) { + sSError(pSender, "unexpected start time in msg. sender startTime:%" PRId64 ", msg startTime:%" PRId64, + pSender->startTime, pMsg->startTime); + goto _ERROR; + } + + ASSERT(pMsg->startTime == pSender->startTime); + // state, term, seq/ack if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender not leader"); @@ -1039,20 +1066,12 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { goto _ERROR; } - if (pMsg->startTime != pSender->startTime) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver time not match"); - sSError(pSender, "sender:%" PRId64 " receiver:%" PRId64 " time not match, error:%s 0x%x", pMsg->startTime, - pSender->startTime, tstrerror(pMsg->code), pMsg->code); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - goto _ERROR; - } - SyncTerm currentTerm = raftStoreGetTerm(pSyncNode); if (pMsg->term != currentTerm) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver term not match"); sSError(pSender, "snapshot sender term not equal, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term, currentTerm); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; goto _ERROR; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 383e4e9d8a..4cc86d51b7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -440,6 +440,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encod // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_SIGNATURE, "Sync signature mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync leader is unreachable") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEW_CONFIG_ERROR, "Sync new config error") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready to propose") From e901adfdf3653c224ffc24da86bd4bafc560ebff Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 20 Sep 2023 14:27:05 +0800 Subject: [PATCH 111/177] enh: check snapshot receiver and sender by signature --- source/libs/sync/src/syncSnapshot.c | 79 ++++++++++++++++++----------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 383fda89b0..59b33d20cc 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -79,8 +79,6 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { int8_t started = atomic_val_compare_exchange_8(&pSender->start, false, true); if (started) return 0; - taosMsleep(1); - pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; pSender->pReader = NULL; @@ -394,6 +392,14 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return (pReceiver != NULL ? atomic_load_8(&pReceiver->start) : false); } +static int32_t snapshotReceiverSignatureCmp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { + if (pReceiver->term < pMsg->term) return -1; + if (pReceiver->term > pMsg->term) return 1; + if (pReceiver->startTime < pMsg->startTime) return -1; + if (pReceiver->startTime > pMsg->startTime) return 1; + return 0; +} + static int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { if (pReceiver->pWriter != NULL) { sRError(pReceiver, "vgId:%d, snapshot receiver writer is not null", pReceiver->pSyncNode->vgId); @@ -434,7 +440,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p if (started) return; pReceiver->ack = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; - pReceiver->term = raftStoreGetTerm(pReceiver->pSyncNode); + pReceiver->term = pPreMsg->term; pReceiver->fromId = pPreMsg->srcId; pReceiver->startTime = pPreMsg->startTime; ASSERT(pReceiver->startTime); @@ -585,18 +591,25 @@ static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pM if (snapshotReceiverIsStart(pReceiver)) { // already start - if (pMsg->startTime > pReceiver->startTime) { - sRInfo(pReceiver, "snapshot receiver startTime:%" PRId64 " > msg startTime:%" PRId64 " start receiver", - pReceiver->startTime, pMsg->startTime); + int32_t order = 0; + if ((order = snapshotReceiverSignatureCmp(pReceiver, pMsg)) < 0) { + sRInfo(pReceiver, + "received a new snapshot preparation. restart receiver" + "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", + pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime); goto _START_RECEIVER; - } else if (pMsg->startTime == pReceiver->startTime) { - sRInfo(pReceiver, "snapshot receiver startTime:%" PRId64 " == msg startTime:%" PRId64 " send reply", - pReceiver->startTime, pMsg->startTime); + } else if (order == 0) { + sRInfo(pReceiver, + "received a duplicate snapshot preparation. send reply" + "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", + pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime); goto _SEND_REPLY; } else { // ignore - sRError(pReceiver, "snapshot receiver startTime:%" PRId64 " < msg startTime:%" PRId64 " ignore", - pReceiver->startTime, pMsg->startTime); + sRError(pReceiver, + "received a stale snapshot preparation. ignore" + "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", + pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; code = terrno; goto _SEND_REPLY; @@ -703,10 +716,9 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p goto _SEND_REPLY; } - if (pReceiver->startTime != pMsg->startTime) { - sRError(pReceiver, "snapshot receiver begin failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, - pReceiver->startTime, pMsg->startTime); + if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + sRError(pReceiver, "snapshot receiver begin failed since %s", terrstr()); goto _SEND_REPLY; } @@ -759,10 +771,9 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend int64_t timeNow = taosGetTimestampMs(); int32_t code = 0; - if (pReceiver->startTime != pMsg->startTime) { - sRError(pReceiver, "snapshot receive failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, - pReceiver->startTime, pMsg->startTime); + if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + sRError(pReceiver, "snapshot receive failed since %s.", terrstr()); code = terrno; goto _SEND_REPLY; } @@ -811,7 +822,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs int64_t timeNow = taosGetTimestampMs(); int32_t code = 0; - if (pReceiver->startTime != pMsg->startTime) { + if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { sRError(pReceiver, "snapshot end failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, pReceiver->startTime, pMsg->startTime); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; @@ -880,13 +891,13 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "not in my config"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } if (pMsg->term < raftStoreGetTerm(pSyncNode)) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } @@ -1015,6 +1026,14 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend return 0; } +static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + if (pSender->term < pMsg->term) return -1; + if (pSender->term > pMsg->term) return 1; + if (pSender->startTime < pMsg->startTime) return -1; + if (pSender->startTime > pMsg->startTime) return 1; + return 0; +} + // sender on message // // condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender @@ -1045,19 +1064,21 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } - if (pMsg->startTime < pSender->startTime) { - sSError(pSender, "ignore stale rsp received. sender startTime:%" PRId64 ", msg startTime:%" PRId64, - pSender->startTime, pMsg->startTime); - terrno = pMsg->code; + // check signature + int32_t order = 0; + if ((order = snapshotSenderSignatureCmp(pSender, pMsg)) > 0) { + sSError(pSender, + "received a stale snapshot rsp. ignore it" + "sender signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")", + pSender->term, pSender->startTime, pMsg->term, pMsg->startTime); + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; - } else if (pMsg->startTime > pSender->startTime) { - sSError(pSender, "unexpected start time in msg. sender startTime:%" PRId64 ", msg startTime:%" PRId64, - pSender->startTime, pMsg->startTime); + } else if (order < 0) { + sSError(pSender, "snapshot sender is stale. stop"); + terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; goto _ERROR; } - ASSERT(pMsg->startTime == pSender->startTime); - // state, term, seq/ack if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender not leader"); From a6d5deb5d14c34d2d77341c9268be137c6bf5a86 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 20 Sep 2023 15:38:34 +0800 Subject: [PATCH 112/177] fixup: fix tsdbFSDoSanAndFix --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index ef2f81fa02..f307411520 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -521,8 +521,9 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { } if (corrupt) { - tsdbError("vgId:%d, not to clear unreferenced files since some fset corrupted", TD_VID(fs->tsdb->pVnode)); + tsdbError("vgId:%d, not to clear unreferenced files due to fset incompleteness", TD_VID(fs->tsdb->pVnode)); fs->fsstate = TSDB_FS_STATE_INCOMPLETE; + code = 0; goto _exit; } From a53ba24118ba94b3fd3d212a6948b3aa69290829 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 20 Sep 2023 15:49:34 +0800 Subject: [PATCH 113/177] fix: set startTime in snapshotReSend --- source/libs/sync/src/syncSnapshot.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 59b33d20cc..1ca93d3844 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -133,7 +133,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; - pMsg->startTime = atomic_load_64(&pSender->startTime); + pMsg->startTime = pSender->startTime; pMsg->seq = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT; if (dataLen > 0) { @@ -270,6 +270,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; + pMsg->startTime = pSender->startTime; pMsg->seq = pSender->seq; if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { From 00aeb031d4bdd87ba46bcab4a891e9358f36b9da Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Sep 2023 09:43:57 +0800 Subject: [PATCH 114/177] enh: replicate a fset if it is corrupted without valid minVer --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index f307411520..cf1158fb01 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -500,7 +500,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { STFileObj *fobj = fset->farr[ftype]; code = tsdbFSDoScanAndFixFile(fs, fobj); if (code) { - fset->maxVerValid = TMIN(fset->maxVerValid, fobj->f->minVer - 1); + fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1; corrupt = true; } } @@ -512,7 +512,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { TARRAY2_FOREACH(lvl->fobjArr, fobj) { code = tsdbFSDoScanAndFixFile(fs, fobj); if (code) { - fset->maxVerValid = TMIN(fset->maxVerValid, fobj->f->minVer - 1); + fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1; corrupt = true; } } From be6411ebbe88dd4bb81f4e9cb084a30ac8aa0e95 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Sep 2023 16:11:06 +0800 Subject: [PATCH 115/177] fixup: set lastSendTime with taosGetTimestampMs --- source/libs/sync/src/syncSnapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 1ca93d3844..2948dbf3d2 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -96,7 +96,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->sendingMS = 0; pSender->term = raftStoreGetTerm(pSender->pSyncNode); pSender->startTime = taosGetMonoTimestampMs(); - pSender->lastSendTime = pSender->startTime; + pSender->lastSendTime = taosGetTimestampMs(); pSender->finish = false; // Get full snapshot info From c89c69f951351dcf73dff363563293e74a5237fe Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Sep 2023 18:38:10 +0800 Subject: [PATCH 116/177] enh: use waitTime to prevent from starting snapshot too frequently --- source/libs/sync/inc/syncSnapshot.h | 2 +- source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncSnapshot.c | 27 +++++++++++++++--------- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 2a19945c5a..95382132b5 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -43,7 +43,7 @@ typedef struct SSyncSnapshotSender { int64_t sendingMS; SyncTerm term; int64_t startTime; - int64_t endTime; + int64_t waitTime; int64_t lastSendTime; bool finish; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 8ae1dd2a54..4776b2bb1b 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -157,7 +157,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) { pReply->fsmState = ths->fsmState; - sError("vgId:%d, not allow to accept sync log msg due to incomplete fsm state", ths->vgId); + sError("vgId:%d, not to accept sync log msg due to incomplete fsm state", ths->vgId); syncEntryDestroy(pEntry); goto _SEND_RESPONSE; } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index b4e2049a64..c57e7e273f 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -72,7 +72,7 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { int32_t syncNodeElect(SSyncNode* pSyncNode) { if (pSyncNode->fsmState == SYNC_FSM_STATE_INCOMPLETE) { - sNError(pSyncNode, "ignore leader hb timeout due to incomplete fsm state"); + sNError(pSyncNode, "skip leader election due to incomplete fsm state"); return -1; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 2948dbf3d2..f9bde9517e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -45,7 +45,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->replicaIndex = replicaIndex; pSender->term = raftStoreGetTerm(pSyncNode); pSender->startTime = -1; - pSender->endTime = -1; + pSender->waitTime = -1; pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); pSender->finish = false; @@ -167,7 +167,7 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { if (stopped) return; pSender->finish = finish; - pSender->endTime = taosGetTimestampMs(); + pSender->waitTime = -1; // close reader if (pSender->pReader != NULL) { @@ -319,8 +319,12 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } - if (pSender->finish && taosGetTimestampMs() - pSender->endTime < SNAPSHOT_WAIT_MS) { - sSDebug(pSender, "snapshot sender start too frequently, ignore"); + int64_t timeNow = taosGetTimestampMs(); + if (pSender->waitTime <= 0) { + pSender->waitTime = timeNow + SNAPSHOT_WAIT_MS; + } + if (timeNow < pSender->waitTime) { + sSDebug(pSender, "snapshot sender waitTime not expired yet, ignore"); return 0; } @@ -674,8 +678,6 @@ _SEND_REPLY: pRspMsg->code = code; pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); - ASSERT(pRspMsg->startTime); - if (snapInfo.data) { pRspMsg->payloadType = snapInfo.type; memcpy(pRspMsg->data, snapInfo.data, dataLen); @@ -684,6 +686,8 @@ _SEND_REPLY: SSnapshotParam *pParam = &pReceiver->snapshotParam; void *data = taosMemoryRealloc(pParam->data, dataLen); if (data == NULL) { + sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, + strerror(errno), dataLen); terrno = TSDB_CODE_OUT_OF_MEMORY; code = terrno; goto _out; @@ -695,7 +699,7 @@ _SEND_REPLY: // send msg syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver pre-snapshot"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); + sRError(pReceiver, "failed to send resp since %s", terrstr()); code = terrno; } @@ -916,13 +920,16 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { if (pMsg->term == raftStoreGetTerm(pSyncNode)) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); + sInfo("vgId:%d, receive pre-snapshot msg of snapshot replication. signature:(%" PRId64 ", %" PRId64 ")", + pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotPrep(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq begin"); + sInfo("vgId:%d, receive begin msg of snapshot replication. signature:(%" PRId64 ", %" PRId64 ")", + pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotBegin(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq end"); + sInfo("vgId:%d, receive end msg of snapshot replication. signature: (%" PRId64 ", %" PRId64 ")", + pSyncNode->vgId, pMsg->term, pMsg->startTime); code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); From 810678ebcb59ba8980a5ba9c8c1be22f965e2807 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Sep 2023 19:02:12 +0800 Subject: [PATCH 117/177] enh: tidy up debugging msgs of snapshot replication --- source/libs/sync/src/syncSnapshot.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index f9bde9517e..a90877ddfc 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -697,7 +697,6 @@ _SEND_REPLY: } // send msg - syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver pre-snapshot"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { sRError(pReceiver, "failed to send resp since %s", terrstr()); code = terrno; @@ -717,19 +716,19 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p int32_t code = TSDB_CODE_SYN_INTERNAL_ERROR; if (!snapshotReceiverIsStart(pReceiver)) { - sRError(pReceiver, "snapshot receiver begin failed since not start"); + sRError(pReceiver, "failed to begin snapshot receiver since not started"); goto _SEND_REPLY; } if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; - sRError(pReceiver, "snapshot receiver begin failed since %s", terrstr()); + sRError(pReceiver, "failed to begin snapshot receiver since %s", terrstr()); goto _SEND_REPLY; } // start writer if (snapshotReceiverStartWriter(pReceiver, pMsg) != 0) { - sRError(pReceiver, "snapshot receiver begin failed since start writer failed"); + sRError(pReceiver, "failed to start snapshot writer since %s", terrstr()); goto _SEND_REPLY; } @@ -742,7 +741,7 @@ _SEND_REPLY: // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { - sRError(pReceiver, "snapshot receiver build resp failed since %s", terrstr()); + sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); return -1; } @@ -757,12 +756,9 @@ _SEND_REPLY: pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; - ASSERT(pRspMsg->startTime); - // send msg - syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver begin"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "snapshot receiver send resp failed since %s", terrstr()); + sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr()); return -1; } @@ -778,7 +774,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; - sRError(pReceiver, "snapshot receive failed since %s.", terrstr()); + sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr()); code = terrno; goto _SEND_REPLY; } @@ -794,7 +790,7 @@ _SEND_REPLY: // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { - sRError(pReceiver, "snapshot receiver build resp failed since %s", terrstr()); + sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); return -1; } @@ -809,11 +805,9 @@ _SEND_REPLY: pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; - ASSERT(pRspMsg->startTime); // send msg - syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver received"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "snapshot receiver send resp failed since %s", terrstr()); + sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr()); return -1; } From 233a3c403444ba77727d41b996f35601295e936d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Sep 2023 20:06:30 +0800 Subject: [PATCH 118/177] enh: terminate on incompletenss of fsm state for dbs of single replica --- source/libs/sync/src/syncMain.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8ddd55d906..eaaccecf90 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1010,8 +1010,11 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { sNTrace(pSyncNode, "reset commit index by snapshot"); } pSyncNode->fsmState = snapshot.state; - if (pSyncNode->fsmState) { - sError("vgId:%d, fsm state incomplete.", pSyncNode->vgId); + if (pSyncNode->fsmState != SYNC_FSM_STATE_NORMAL) { + sError("vgId:%d, fsm state is incomplete.", pSyncNode->vgId); + if (pSyncNode->replicaNum == 1) { + goto _error; + } } } pSyncNode->commitIndex = commitIndex; From 9643799dc5712aa335fc752b5095df4e3edf746b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 22 Sep 2023 08:53:14 +0800 Subject: [PATCH 119/177] fixup: compiler error at labels since allowed only at statements --- source/libs/sync/src/syncSnapshot.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a90877ddfc..924813eb98 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -786,7 +786,8 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend } } -_SEND_REPLY: +_SEND_REPLY:; + // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { @@ -834,7 +835,8 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs snapshotReceiverStop(pReceiver); } -_SEND_REPLY: +_SEND_REPLY:; + // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { From 29bbebc3238fbf420bc119d6f6d8690097a62553 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 27 Sep 2023 09:29:38 +0800 Subject: [PATCH 120/177] enh: refactor a func name as tsdbSnapGetDetails --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 13 ++++++++----- source/dnode/vnode/src/vnd/vnodeSync.c | 7 ++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d5e1585af4..eaffa2500d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -295,7 +295,7 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback); // STsdbSnapReader ======================================== -int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap); +int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap); int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges, STsdbSnapReader** ppReader); int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6ee7112906..5fcf384bfb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1507,13 +1507,15 @@ void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { ppList[0] = NULL; } -int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { - pSnap->state = pTsdb->pFS->fsstate; - if (pSnap->type != TDMT_SYNC_PREP_SNAPSHOT && pSnap->type != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - return 0; +int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { + int code = -1; + if (pVnode->pTsdb->pFS->fsstate == TSDB_FS_STATE_NORMAL) { + pSnap->state = SYNC_FSM_STATE_NORMAL; + } else { + pSnap->state = SYNC_FSM_STATE_INCOMPLETE; } - int code = -1; + STsdb* pTsdb = pVnode->pTsdb; STsdbSnapPartList* pList = tsdbGetSnapPartList(pTsdb->pFS); if (pList == NULL) goto _out; @@ -1522,6 +1524,7 @@ int32_t tsdbSnapGetInfo(STsdb* pTsdb, SSnapshot* pSnap) { void* buf = NULL; int32_t tlen = 0; + // estimate data length encode int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY bufLen += sizeof(SSyncTLV); // subtyp: SNAP_DATA_TSDB diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ba142ddb6d..b424e28f72 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -787,12 +787,13 @@ bool vnodeIsLeader(SVnode *pVnode) { } int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { + int code = 0; pSnap->lastApplyIndex = pVnode->state.committed; pSnap->lastApplyTerm = pVnode->state.commitTerm; pSnap->lastConfigIndex = -1; - int32_t code = tsdbSnapGetInfo(pVnode->pTsdb, pSnap); - - pSnap->state = (pSnap->state == TSDB_FS_STATE_INCOMPLETE) ? SYNC_FSM_STATE_INCOMPLETE : SYNC_FSM_STATE_NORMAL; + if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + code = tsdbSnapGetDetails(pVnode, pSnap); + } return code; } From b80f8201cea02020d3b5c071b11fe3bb0f9a07c8 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 27 Sep 2023 15:22:24 +0800 Subject: [PATCH 121/177] feat: detect fs incompleteness of RSMA tsdbs --- source/dnode/vnode/src/inc/tsdb.h | 8 ++++++-- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 18 ++++++++++++------ source/dnode/vnode/src/vnd/vnodeSync.c | 5 +++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 1d17d616ab..ec721308b2 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -1025,10 +1025,14 @@ struct STsdbFilterInfo { TABLEID tbid; }; -enum { +typedef enum { TSDB_FS_STATE_NORMAL = 0, TSDB_FS_STATE_INCOMPLETE, -}; +} ETsdbFsState; + +// utils +ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode); +int32_t tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index eaffa2500d..6682ff0133 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -295,7 +295,6 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback); // STsdbSnapReader ======================================== -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap); int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges, STsdbSnapReader** ppReader); int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader); @@ -499,6 +498,7 @@ struct SSma { #define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb) #define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L0]) #define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L1]) +#define SMA_RSMA_GET_TSDB(pVnode, level) ((level == 0) ? pVnode->pTsdb : pVnode->pSma->pRSmaTsdb[level - 1]) // sma void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 5fcf384bfb..69ead6feef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1507,14 +1507,20 @@ void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { ppList[0] = NULL; } -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { - int code = -1; - if (pVnode->pTsdb->pFS->fsstate == TSDB_FS_STATE_NORMAL) { - pSnap->state = SYNC_FSM_STATE_NORMAL; - } else { - pSnap->state = SYNC_FSM_STATE_INCOMPLETE; +ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { + if (!VND_IS_RSMA(pVnode)) { + return pVnode->pTsdb->pFS->fsstate; } + for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { + if (SMA_RSMA_GET_TSDB(pVnode, lvl)->pFS->fsstate != TSDB_FS_STATE_NORMAL) { + return TSDB_FS_STATE_INCOMPLETE; + } + } + return TSDB_FS_STATE_NORMAL; +} +int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { + int code = -1; STsdb* pTsdb = pVnode->pTsdb; STsdbSnapPartList* pList = tsdbGetSnapPartList(pTsdb->pFS); if (pList == NULL) goto _out; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b424e28f72..86fb7cb55b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -791,6 +791,11 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { pSnap->lastApplyIndex = pVnode->state.committed; pSnap->lastApplyTerm = pVnode->state.commitTerm; pSnap->lastConfigIndex = -1; + pSnap->state = SYNC_FSM_STATE_NORMAL; + + if (tsdbSnapGetFsState(pVnode) != TSDB_FS_STATE_NORMAL) { + pSnap->state = SYNC_FSM_STATE_INCOMPLETE; + } if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { code = tsdbSnapGetDetails(pVnode, pSnap); From d5d713b1b40f0d6300a5de850b83209ebacf163f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 27 Sep 2023 17:25:56 +0800 Subject: [PATCH 122/177] enh: fill snapshot info for rsma tsdbs --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 66 +++++++++++++--------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index cf1158fb01..813123ae5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -521,7 +521,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { } if (corrupt) { - tsdbError("vgId:%d, not to clear unreferenced files due to fset incompleteness", TD_VID(fs->tsdb->pVnode)); + tsdbError("vgId:%d, not to clear dangling files due to fset incompleteness", TD_VID(fs->tsdb->pVnode)); fs->fsstate = TSDB_FS_STATE_INCOMPLETE; code = 0; goto _exit; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 69ead6feef..2cf4521eff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1521,53 +1521,65 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int code = -1; - STsdb* pTsdb = pVnode->pTsdb; - STsdbSnapPartList* pList = tsdbGetSnapPartList(pTsdb->pFS); - if (pList == NULL) goto _out; + int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); + int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; - if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); + pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); + if (pLists[j] == NULL) goto _out; } - void* buf = NULL; - int32_t tlen = 0; - - // estimate data length encode + // estimate bufLen and prepare int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY - bufLen += sizeof(SSyncTLV); // subtyp: SNAP_DATA_TSDB - bufLen += tTsdbSnapPartListDataLenCalc(pList); + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + bufLen += sizeof(SSyncTLV); // subTyps[j] + bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); + } + + tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); void* data = taosMemoryRealloc(pSnap->data, bufLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; } pSnap->data = data; // header - SSyncTLV* datHead = (void*)pSnap->data; - datHead->typ = pSnap->type; - datHead->len = 0; + SSyncTLV* head = data; + head->len = 0; + head->typ = pSnap->type; + int32_t offset = sizeof(SSyncTLV); - // tsdb - SSyncTLV* tsdbHead = (void*)datHead->val; - tsdbHead->typ = SNAP_DATA_TSDB; + // fill snapshot info + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + } - buf = tsdbHead->val; - tlen = 0; - if ((tlen = tSerializeTsdbSnapPartList(buf, bufLen, pList)) < 0) { - tsdbError("vgId:%d, failed to serialize snap range since %s", TD_VID(pTsdb->pVnode), terrstr()); - goto _out; + // subHead + SSyncTLV* subHead = (void*)((char*)data + offset); + subHead->typ = subTyps[j]; + ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); + + int32_t tlen = 0; + if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { + tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); + goto _out; + } + subHead->len = tlen; + offset += sizeof(SSyncTLV) + tlen; } - tsdbHead->len = tlen; - datHead->len += sizeof(SSyncTLV) + tsdbHead->len; - - // rsma + head->len = offset; code = 0; _out: - if (pList) { - tsdbSnapPartListDestroy(&pList); + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + if (pLists[j] == NULL) continue; + tsdbSnapPartListDestroy(&pLists[j]); } return code; From 81b7093a42a14ef1f9188e8810c3f141eabac08a Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 Oct 2023 11:17:00 +0800 Subject: [PATCH 123/177] enhance: enhance test --- tests/script/tsim/query/unionall_as_table.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim index 11fdf17978..a03ecac34a 100644 --- a/tests/script/tsim/query/unionall_as_table.sim +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -61,4 +61,5 @@ print $data00 if $data00 != 1 then return -1 endi +sql_error select f from (select f, f from ctcount); system sh/exec.sh -n dnode1 -s stop -x SIGINT From 74185b8b9c393c79bc8a26a78268fcca9a20b0ac Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 7 Oct 2023 16:07:21 +0800 Subject: [PATCH 124/177] feat: exchange snapshot info for rsma tsdbs --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaSnapshot.c | 4 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 106 ++++++++++++--------- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6682ff0133..5015202865 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -357,7 +357,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader); int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== -int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void* pRanges, SRSmaSnapWriter** ppWriter); +int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRanges, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index ca46b4728f..1e921b23d1 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -128,7 +128,7 @@ struct SRSmaSnapWriter { STsdbSnapWriter* pDataWriter[TSDB_RETENTION_L2]; }; -int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void* pRanges, SRSmaSnapWriter** ppWriter) { +int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRanges, SRSmaSnapWriter** ppWriter) { int32_t code = 0; int32_t lino = 0; SVnode* pVnode = pSma->pVnode; @@ -147,7 +147,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void* pRanges // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, pRanges, &pWriter->pDataWriter[i]); + code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, ((void**)ppRanges)[i], &pWriter->pDataWriter[i]); TSDB_CHECK_CODE(code, lino, _exit); } } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 0874e5e0d8..f5a5249d1f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -45,7 +45,7 @@ struct SVSnapReader { SStreamStateReader *pStreamStateReader; // rsma int8_t rsmaDone; - TSnapRangeArray *pRsmaRanges; + TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapReader *pRsmaReader; }; @@ -69,6 +69,20 @@ _out: return code; } +static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { + _Static_assert(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); + switch (tsdbTyp) { + case SNAP_DATA_TSDB: + return &pReader->pRanges; + case SNAP_DATA_RSMA1: + return &pReader->pRsmaRanges[0]; + case SNAP_DATA_RSMA2: + return &pReader->pRsmaRanges[1]; + default: + return NULL; + } +} + int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader) { int32_t code = 0; int64_t sver = pParam->start; @@ -92,25 +106,18 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader goto _err; } - int32_t offset = 0; TSnapRangeArray **ppRanges = NULL; + int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { SSyncTLV *subField = (void *)(datHead->val + offset); offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - switch (subField->typ) { - case SNAP_DATA_TSDB: - ppRanges = &pReader->pRanges; - break; - case SNAP_DATA_RSMA1: - ppRanges = &pReader->pRsmaRanges; - break; - default: - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), - subField->typ); - goto _err; + ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _err; } if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); @@ -129,8 +136,19 @@ _err: return code; } +static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { + int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { + TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); + if (ppRanges == NULL) continue; + tsdbSnapRangeArrayDestroy(ppRanges); + } +} + void vnodeSnapReaderClose(SVSnapReader *pReader) { vInfo("vgId:%d, close vnode snapshot reader", TD_VID(pReader->pVnode)); + vnodeSnapReaderDestroyTsdbRanges(pReader); + if (pReader->pRsmaReader) { rsmaSnapReaderClose(&pReader->pRsmaReader); } @@ -155,14 +173,6 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader); } - if (pReader->pRanges) { - tsdbSnapRangeArrayDestroy(&pReader->pRanges); - } - - if (pReader->pRsmaRanges) { - tsdbSnapRangeArrayDestroy(&pReader->pRsmaRanges); - } - taosMemoryFree(pReader); } @@ -442,10 +452,24 @@ struct SVSnapWriter { SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; // rsma - TSnapRangeArray *pRsmaRanges; + TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapWriter *pRsmaSnapWriter; }; +TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { + _Static_assert(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size"); + switch (tsdbTyp) { + case SNAP_DATA_TSDB: + return &pWriter->pRanges; + case SNAP_DATA_RSMA1: + return &pWriter->pRsmaRanges[0]; + case SNAP_DATA_RSMA2: + return &pWriter->pRsmaRanges[1]; + default: + return NULL; + } +} + int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t code = 0; SVSnapWriter *pWriter = NULL; @@ -477,25 +501,18 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter goto _err; } - int32_t offset = 0; TSnapRangeArray **ppRanges = NULL; + int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { SSyncTLV *subField = (void *)(datHead->val + offset); offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - switch (subField->typ) { - case SNAP_DATA_TSDB: - ppRanges = &pWriter->pRanges; - break; - case SNAP_DATA_RSMA1: - ppRanges = &pWriter->pRsmaRanges; - break; - default: - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), - subField->typ); - goto _err; + ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _err; } if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); @@ -515,10 +532,21 @@ _err: return code; } +static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) { + int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { + TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); + if (ppRanges == NULL) continue; + tsdbSnapRangeArrayDestroy(ppRanges); + } +} + int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot) { int32_t code = 0; SVnode *pVnode = pWriter->pVnode; + vnodeSnapWriterDestroyTsdbRanges(pWriter); + // prepare if (pWriter->pTsdbSnapWriter) { tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter); @@ -550,10 +578,6 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } - if (pWriter->pRanges) { - tsdbSnapRangeArrayDestroy(&pWriter->pRanges); - } - if (pWriter->pTsdbSnapWriter) { code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback); if (code) goto _exit; @@ -588,10 +612,6 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } - if (pWriter->pRsmaRanges) { - tsdbSnapRangeArrayDestroy(&pWriter->pRsmaRanges); - } - if (pWriter->pRsmaSnapWriter) { code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback); if (code) goto _exit; @@ -739,7 +759,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { case SNAP_DATA_QTASK: { // rsma1/rsma2/qtask for rsma if (pWriter->pRsmaSnapWriter == NULL) { - code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, pWriter->pRsmaRanges, + code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges, &pWriter->pRsmaSnapWriter); if (code) goto _err; } From c1f1709d59c6b279c6d4f52bde05c16b4924e927 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 8 Oct 2023 17:03:07 +0800 Subject: [PATCH 125/177] fix: add rsmaSnapWriterPrepareClose --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/sma/smaSnapshot.c | 15 +++++++++++++++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 ++++ 3 files changed, 20 insertions(+) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5015202865..68334b3b63 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -359,6 +359,7 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRanges, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); typedef struct { diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 1e921b23d1..c93d9a7de6 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -165,6 +165,21 @@ _exit: return code; } +int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { + int32_t code = 0; + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pWriter->pDataWriter[i]) { + code = tsdbSnapWriterPrepareClose(pWriter->pDataWriter[i]); + if (code) { + smaError("vgId:%d, failed to prepare close tsdbSnapWriter since %s. i: %d", SMA_VID(pWriter->pSma), terrstr(), + i); + return -1; + } + } + } + return code; +} + int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; int32_t lino = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f5a5249d1f..f5d0bf6d0f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -552,6 +552,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter); } + if (pWriter->pRsmaSnapWriter) { + rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter); + } + // commit json if (!rollback) { pWriter->info.state.committed = pWriter->ever; From d671283b8be9412f76027d2913f51d7387cd1aab Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 8 Oct 2023 17:41:33 +0800 Subject: [PATCH 126/177] refact: adjust logging msg for incomplete fsm state --- source/libs/sync/src/syncAppendEntries.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4776b2bb1b..647b86bae9 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -155,9 +155,9 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pEntry->term); - if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) { + if (ths->fsmState != SYNC_FSM_STATE_NORMAL) { pReply->fsmState = ths->fsmState; - sError("vgId:%d, not to accept sync log msg due to incomplete fsm state", ths->vgId); + sWarn("vgId:%d, unable to accept, due to incomplete fsm state. index:%" PRId64, ths->vgId, pEntry->index); syncEntryDestroy(pEntry); goto _SEND_RESPONSE; } From 05ba5e1ed06cfa23d10adbeffed5737e042866e7 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 8 Oct 2023 17:42:20 +0800 Subject: [PATCH 127/177] refact: adjust logging msg in walLogEntriesComplete --- source/libs/wal/src/walMeta.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e700ef3d0a..f5e5427c68 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -339,8 +339,9 @@ bool walLogEntriesComplete(const SWal* pWal) { } if (!complete) { - wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], aligned with snaphotVer:%" PRId64, - pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer); + wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], index:%" PRId64 + ", snaphotVer:%" PRId64, + pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, index, pWal->vers.snapshotVer); terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; } From 094cf408dfdf72ffcda5cc092e0f3a9c6e836f00 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 8 Oct 2023 18:28:47 +0800 Subject: [PATCH 128/177] enh: prepare vnode dir again in vnodeOpen --- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 5084cc2ff5..28d1e171d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -345,6 +345,10 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC return NULL; } + if (vnodeMkDir(pTfs, path)) { + vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", info.config.vgId, strerror(errno), path); + return NULL; + } // save vnode info on dnode ep changed bool updated = false; SSyncCfg *pCfg = &info.config.syncCfg; From 4e74533878bb2a8d6c1b08dc02940ed2052fe04f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 9 Oct 2023 17:19:47 +0800 Subject: [PATCH 129/177] enh: avoid _Static_assert --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f5d0bf6d0f..cdf3fff47f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -70,7 +70,7 @@ _out: } static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { - _Static_assert(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); + ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: return &pReader->pRanges; @@ -457,7 +457,7 @@ struct SVSnapWriter { }; TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { - _Static_assert(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size"); + ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: return &pWriter->pRanges; From eb8c4d3e8c1d2dad9d23a5d50ad5b85424c86b89 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 10 Oct 2023 11:08:12 +0800 Subject: [PATCH 130/177] enh: use hash table for filtering fset in tsdbSnapReaderOpen and tsdbSnapWriterOpen --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS2.c | 72 +++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 13 +++- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ec721308b2..79112babc3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -680,6 +680,7 @@ typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); +SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges); // snap partition list typedef TARRAY2(SVersionRange) SVerRangeList; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 813123ae5c..7b8c8696ea 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -991,11 +991,12 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr, TFileOpArray *fopArr) { int32_t code = 0; STFileSet *fset; STFileSet *fset1; + SHashObj *pHash = NULL; fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray)); if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -1003,21 +1004,19 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclu TARRAY2_INIT(fsetArr[0]); int32_t i = 0; + if (pRanges) { + pHash = tsdbGetSnapRangeHash(pRanges); + } taosThreadRwlockRdlock(&fs->tsdb->rwLock); TARRAY2_FOREACH(fs->fSetArr, fset) { int64_t ever = VERSION_MAX; - while (pExclude && i < TARRAY2_SIZE(pExclude)) { - STSnapRange *u = TARRAY2_GET(pExclude, i); - if (fset->fid > u->fid) { - i++; - continue; - } - if (fset->fid == u->fid) { + if (pHash) { + int32_t fid = fset->fid; + STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + if (u) { ever = u->sver - 1; - i++; } - break; } code = tsdbTFileSetFilteredInitDup(fs->tsdb, fset, ever, &fset1, fopArr); @@ -1033,14 +1032,37 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclu taosMemoryFree(fsetArr[0]); fsetArr[0] = NULL; } + if (pHash) { + taosHashCleanup(pHash); + pHash = NULL; + } return code; } +SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { + int32_t capacity = TARRAY2_SIZE(pRanges) * 2; + SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (pHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { + STSnapRange *u = TARRAY2_GET(pRanges, i); + int32_t fid = u->fid; + int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); + ASSERT(code == 0); + tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); + } + return pHash; +} + int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, TSnapRangeArray **fsrArr) { int32_t code = -1; STFileSet *fset; STSnapRange *fsr1 = NULL; + SHashObj *pHash = NULL; fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); if (fsrArr[0] == NULL) { @@ -1048,30 +1070,32 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev goto _out; } - int32_t i = 0; + tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges))); code = 0; + if (pRanges) { + pHash = tsdbGetSnapRangeHash(pRanges); + } taosThreadRwlockRdlock(&fs->tsdb->rwLock); TARRAY2_FOREACH(fs->fSetArr, fset) { int64_t sver1 = sver; int64_t ever1 = ever; - while (pRanges && i < TARRAY2_SIZE(pRanges)) { - STSnapRange *u = TARRAY2_GET(pRanges, i); - if (fset->fid > u->fid) { - i++; - continue; - } - if (fset->fid == u->fid) { + if (pHash) { + int32_t fid = fset->fid; + STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + if (u) { sver1 = u->sver; - i++; + tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); } - break; } - if (sver1 > ever1) continue; + if (sver1 > ever1) { + tsdbDebug("skip fid:%d, sver:%" PRId64 ", ever:%" PRId64, fset->fid, sver1, ever1); + continue; + } - tsdbInfo("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); + tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); if (code) break; @@ -1090,6 +1114,10 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev } _out: + if (pHash) { + taosHashCleanup(pHash); + pHash = NULL; + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 2cf4521eff..d348c318b7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1175,6 +1175,12 @@ static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { return 0; } +static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + STsdbSnapPartition* tsdbSnapPartitionCreate() { STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); if (pSP == NULL) { @@ -1478,10 +1484,13 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** r->fid = part->fid; r->sver = maxVerValid + 1; r->ever = VERSION_MAX; - tsdbInfo("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - TARRAY2_APPEND(pDiff, r); + tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn); + ASSERT(code == 0); } ppRanges[0] = pDiff; + + tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); return 0; _err: From 59e8a2104c226884ea8e62b27794ad3a428de595 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 10 Oct 2023 20:19:07 +0800 Subject: [PATCH 131/177] refact: wrap funcs to handle snapshot info for vnodeSnapReaderOpen and vnodeSnapWriterOpen --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 126 +++++++++++++-------- 1 file changed, 76 insertions(+), 50 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index cdf3fff47f..87b407efcb 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -83,6 +83,42 @@ static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int } } +static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { + SVnode *pVnode = pReader->pVnode; + int32_t code = -1; + + if (pParam->data) { + SSyncTLV *datHead = (void *)pParam->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + + TSnapRangeArray **ppRanges = NULL; + int32_t offset = 0; + + while (offset + sizeof(SSyncTLV) < datHead->len) { + SSyncTLV *subField = (void *)(datHead->val + offset); + offset += sizeof(SSyncTLV) + subField->len; + void *buf = subField->val; + int32_t bufLen = subField->len; + ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } + } + + code = 0; +_out: + return code; +} + int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader) { int32_t code = 0; int64_t sver = pParam->start; @@ -99,31 +135,8 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->ever = ever; // snapshot info - if (pParam->data) { - SSyncTLV *datHead = (void *)pParam->data; - if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - terrno = TSDB_CODE_INVALID_DATA_FMT; - goto _err; - } - - TSnapRangeArray **ppRanges = NULL; - int32_t offset = 0; - - while (offset + sizeof(SSyncTLV) < datHead->len) { - SSyncTLV *subField = (void *)(datHead->val + offset); - offset += sizeof(SSyncTLV) + subField->len; - void *buf = subField->val; - int32_t bufLen = subField->len; - ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _err; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _err; - } - } + if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) { + goto _err; } vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); @@ -470,6 +483,42 @@ TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t ts } } +static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { + SVnode *pVnode = pWriter->pVnode; + int32_t code = -1; + + if (pParam->data) { + SSyncTLV *datHead = (void *)pParam->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + + TSnapRangeArray **ppRanges = NULL; + int32_t offset = 0; + + while (offset + sizeof(SSyncTLV) < datHead->len) { + SSyncTLV *subField = (void *)(datHead->val + offset); + offset += sizeof(SSyncTLV) + subField->len; + void *buf = subField->val; + int32_t bufLen = subField->len; + ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } + } + + code = 0; +_out: + return code; +} + int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t code = 0; SVSnapWriter *pWriter = NULL; @@ -494,31 +543,8 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter pWriter->commitID = ++pVnode->state.commitID; // snapshot info - if (pParam->data) { - SSyncTLV *datHead = (void *)pParam->data; - if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - terrno = TSDB_CODE_INVALID_DATA_FMT; - goto _err; - } - - TSnapRangeArray **ppRanges = NULL; - int32_t offset = 0; - - while (offset + sizeof(SSyncTLV) < datHead->len) { - SSyncTLV *subField = (void *)(datHead->val + offset); - offset += sizeof(SSyncTLV) + subField->len; - void *buf = subField->val; - int32_t bufLen = subField->len; - ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _err; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _err; - } - } + if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) { + goto _err; } vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode), From 83013e0fe518827ccbfed805abce27d73d59b3f9 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 11 Oct 2023 15:17:32 +0800 Subject: [PATCH 132/177] enh: return error in case of out of memory for tsdbGetSnapRangeHash --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 7b8c8696ea..4df5a1eeec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1000,12 +1000,14 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray)); if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY; - TARRAY2_INIT(fsetArr[0]); - int32_t i = 0; if (pRanges) { pHash = tsdbGetSnapRangeHash(pRanges); + if (pHash == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } } taosThreadRwlockRdlock(&fs->tsdb->rwLock); @@ -1027,6 +1029,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange } taosThreadRwlockUnlock(&fs->tsdb->rwLock); +_out: if (code) { TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); taosMemoryFree(fsetArr[0]); @@ -1059,7 +1062,7 @@ SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, TSnapRangeArray **fsrArr) { - int32_t code = -1; + int32_t code = 0; STFileSet *fset; STSnapRange *fsr1 = NULL; SHashObj *pHash = NULL; @@ -1071,9 +1074,12 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev } tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges))); - code = 0; if (pRanges) { pHash = tsdbGetSnapRangeHash(pRanges); + if (pHash == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } } taosThreadRwlockRdlock(&fs->tsdb->rwLock); From f2bd43c07e9f1302ccfe3fb29c580ddbda0b196f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 13 Oct 2023 20:46:53 +0800 Subject: [PATCH 133/177] fix: set field length properly in tsdbGetSnapDetails --- include/common/tgrant.h | 6 +++--- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index edbc74bf18..cfc6c13c48 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -52,9 +52,9 @@ typedef enum { int32_t grantCheck(EGrantType grant); #ifndef TD_GRANT_OPTIMIZE -int32_t grantAlterActiveCode(const char* old, const char* new, char* out, int8_t type); +int32_t grantAlterActiveCode(const char* old, const char* newer, char* out, int8_t type); #else -int32_t grantAlterActiveCode(int32_t did, const char* old, const char* new, char* out, int8_t type); +int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); #endif #ifndef GRANTS_CFG @@ -114,4 +114,4 @@ int32_t grantAlterActiveCode(int32_t did, const char* old, const char* new, char } #endif -#endif /*_TD_COMMON_GRANT_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_GRANT_H_*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index d348c318b7..3b4827a6be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1562,6 +1562,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { head->len = 0; head->typ = pSnap->type; int32_t offset = sizeof(SSyncTLV); + int32_t tlen = 0; // fill snapshot info for (int32_t j = 0; j < tsdbMaxCnt; ++j) { @@ -1573,7 +1574,6 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { subHead->typ = subTyps[j]; ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - int32_t tlen = 0; if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); goto _out; @@ -1582,7 +1582,8 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { offset += sizeof(SSyncTLV) + tlen; } - head->len = offset; + head->len = offset - sizeof(SSyncTLV); + ASSERT(offset <= bufLen); code = 0; _out: From 5c85525fd07f9cd8b6a805928ea848d80bc6bd18 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 16 Oct 2023 11:21:30 +0800 Subject: [PATCH 134/177] enh: add tsdbFSCreateRefSnapshotWithoutLock --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 9 ++++++++- source/dnode/vnode/src/tsdb/tsdbFS2.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 4df5a1eeec..93a16b5502 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -961,6 +961,13 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) { } int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + int32_t code = tsdbFSCreateRefSnapshotWithoutLock(fs, fsetArr); + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + return code; +} + +int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr) { int32_t code = 0; STFileSet *fset, *fset1; @@ -1284,4 +1291,4 @@ int32_t tsdbFSEnableBgTask(STFileSystem *fs) { fs->stop = false; taosThreadMutexUnlock(fs->mutex); return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 851459df53..31b98e5656 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -52,6 +52,7 @@ int32_t tsdbCloseFS(STFileSystem **fs); int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); +int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1139524cb3..22d2a2098c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4947,7 +4947,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs } // fs - code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray); + code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray); // unlock taosThreadRwlockUnlock(&pTsdb->rwLock); From c4e9069a664f53d282e165866582c3c43b893524 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 16 Oct 2023 16:36:38 +0800 Subject: [PATCH 135/177] fix: set nextProcessedVer properly in tqProcessTaskScanHistory --- source/dnode/vnode/src/tq/tq.c | 5 ++++- source/libs/stream/src/streamExec.c | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 98464d082c..a5832d3c66 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1200,6 +1200,9 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { pStreamTask->status.taskStatus = TASK_STATUS__HALT; nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); + if (nextProcessedVer == -1) { + nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1; + } tqDebug("s-task:%s level:%d nextProcessedVer:%" PRId64 ", sched-status:%d is halt by fill-history task:%s", pStreamTask->id.idStr, pStreamTask->info.taskLevel, nextProcessedVer, pStreamTask->status.schedStatus, @@ -1975,4 +1978,4 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 12b51e6c93..c49c647906 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -309,7 +309,9 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { pStreamTask->id.idStr); } - ASSERT(pStreamTask->hTaskInfo.id.taskId == pTask->id.taskId && pTask->status.appendTranstateBlock == true); + ASSERT(((pStreamTask->status.taskStatus == TASK_STATUS__STOP) || + (pStreamTask->hTaskInfo.id.taskId == pTask->id.taskId)) && + pTask->status.appendTranstateBlock == true); STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; From e463e0690d23debc2d98fe105b487b51d1fac758 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 17 Oct 2023 16:02:22 +0800 Subject: [PATCH 136/177] enh: check existence of files properly in multilevel.py --- tests/system-test/0-others/multilevel.py | 43 ++++++++++++++++++------ 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 66434fff67..def2c3152b 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -17,6 +17,28 @@ from util.cases import * from util.sql import * from util.common import * from util.sqlset import * +import glob + +def scanFiles(pattern): + res = [] + for f in glob.iglob(pattern): + res += [f] + return res + +def checkFiles(pattern, state): + res = scanFiles(pattern) + tdLog.info(res) + num = len(res) + if num: + if state: + tdLog.info("%s: %d files exist. expect: files exist" % (pattern, num)) + else: + tdLog.exit("%s: %d files exist. expect: files not exist." % (pattern, num)) + else: + if state: + tdLog.exit("%s: %d files exist. expect: files exist" % (pattern, num)) + else: + tdLog.info("%s: %d files exist. expect: files not exist." % (pattern, num)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): @@ -41,8 +63,8 @@ class TDTestCase: tdDnodes.start(1) tdLog.info("================= step2") - tdSql.haveFile('/mnt/data1/',1) - tdSql.haveFile('/mnt/data2/',0) + checkFiles(r'/mnt/data1/*/*',1) + checkFiles(r'/mnt/data2/*/*',0) tdDnodes.stop(1) def dir_not_exist(self): tdLog.info("============== dir_not_exist test ===============") @@ -156,9 +178,9 @@ class TDTestCase: tdDnodes.start(1) for i in dir_list: if i == '/mnt/data000': - tdSql.haveFile(i,1) + checkFiles("%s/*/*" % i, 1) else: - tdSql.haveFile(i,0) + checkFiles("%s/*/*" % i, 0) def more_than_16_disks(self): tdLog.info("============== more_than_16_disks test ===============") @@ -223,7 +245,8 @@ class TDTestCase: for i in range(10,30): tdSql.execute(f'insert into tb1 values(now-{i}d,10)') tdSql.execute('flush database dbtest') - tdSql.haveFile('/mnt/data1/',1) + time.sleep(3) + checkFiles('/mnt/data1/vnode/*/tsdb/v*',1) tdDnodes.stop(1) cfg={ '/mnt/data1 0 1' : 'dataDir', @@ -234,14 +257,14 @@ class TDTestCase: tdSql.createDir('/mnt/data3') tdDnodes.deploy(1,cfg) tdDnodes.start(1) - tdSql.haveFile('/mnt/data1/',1) - tdSql.haveFile('/mnt/data2/',0) - tdSql.haveFile('/mnt/data3/',0) + checkFiles('/mnt/data1/vnode/*/tsdb/v*',1) + checkFiles('/mnt/data2/vnode/*/tsdb/v*',0) + checkFiles('/mnt/data3/vnode/*/tsdb/v*',0) tdSql.execute('alter database dbtest keep 10d,365d,3650d') tdSql.execute('trim database dbtest') time.sleep(3) - tdSql.haveFile('/mnt/data1/',1) - tdSql.haveFile('/mnt/data2/',1) + checkFiles('/mnt/data1/vnode/*/tsdb/v*',1) + checkFiles('/mnt/data2/vnode/*/tsdb/v*',1) def run(self): self.basic() From a95f6e686287eea552faccae5d4ef216b29c1864 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 11 Oct 2023 20:49:43 +0800 Subject: [PATCH 137/177] feat: support restore dnode with vnodes of replaced disks for primary dirs --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 37 +++++++++++++-------- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 32 +++++++++++++----- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 10 +++--- source/dnode/vnode/src/vnd/vnodeOpen.c | 7 +++- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index cddf132bce..34f2b5c446 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -56,6 +56,7 @@ typedef struct { int32_t vgVersion; int32_t refCount; int8_t dropped; + int8_t failed; int8_t disable; int32_t diskPrimary; int32_t toVgId; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index b4fe824466..3d7f2b9e9e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -30,9 +30,11 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) { if (ppVnode == NULL || *ppVnode == NULL) continue; SVnodeObj *pVnode = *ppVnode; - SVnodeLoad vload = {0}; - vnodeGetLoad(pVnode->pImpl, &vload); - if (isReset) vnodeResetLoad(pVnode->pImpl, &vload); + SVnodeLoad vload = {.vgId = pVnode->vgId}; + if (!pVnode->failed) { + vnodeGetLoad(pVnode->pImpl, &vload); + if (isReset) vnodeResetLoad(pVnode->pImpl, &vload); + } taosArrayPush(pInfo->pVloads, &vload); pIter = taosHashIterate(pMgmt->hash, pIter); } @@ -52,9 +54,11 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { if (ppVnode == NULL || *ppVnode == NULL) continue; SVnodeObj *pVnode = *ppVnode; - SVnodeLoadLite vload = {0}; - if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { - taosArrayPush(pInfo->pVloads, &vload); + if (!pVnode->failed) { + SVnodeLoadLite vload = {0}; + if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { + taosArrayPush(pInfo->pVloads, &vload); + } } pIter = taosHashIterate(pMgmt->hash, pIter); } @@ -278,7 +282,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { vmGenerateWrapperCfg(pMgmt, &req, &wrapperCfg); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode != NULL) { + if (pVnode != NULL && !pVnode->failed) { dError("vgId:%d, already exist", req.vgId); tFreeSCreateVnodeReq(&req); vmReleaseVnode(pMgmt, pVnode); @@ -287,7 +291,9 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } - wrapperCfg.diskPrimary = vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId); + ASSERT(pVnode == NULL || pVnode->failed); + + wrapperCfg.diskPrimary = pVnode ? pVnode->diskPrimary : vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId); int32_t diskPrimary = wrapperCfg.diskPrimary; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); @@ -364,9 +370,10 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { TMSG_INFO(pMsg->msgType)); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode == NULL) { + if (pVnode == NULL || pVnode->failed) { dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; + if (pVnode) vmReleaseVnode(pMgmt, pVnode); return -1; } @@ -481,9 +488,10 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { req.vgId, TMSG_INFO(pMsg->msgType)); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode == NULL) { + if (pVnode == NULL || pVnode->failed) { dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; + if (pVnode) vmReleaseVnode(pMgmt, pVnode); return -1; } @@ -523,9 +531,10 @@ int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, vnode write disable:%d", req.vgId, req.disable); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode == NULL) { + if (pVnode == NULL || pVnode->failed) { dError("vgId:%d, failed to disable write since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; + if (pVnode) vmReleaseVnode(pMgmt, pVnode); return -1; } @@ -555,9 +564,10 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd, req.dstVgId); pVnode = vmAcquireVnode(pMgmt, srcVgId); - if (pVnode == NULL) { + if (pVnode == NULL || pVnode->failed) { dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; + if (pVnode) vmReleaseVnode(pMgmt, pVnode); return -1; } @@ -669,9 +679,10 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); - if (pVnode == NULL) { + if (pVnode == NULL || pVnode->failed) { dError("vgId:%d, failed to alter replica since %s", vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; + if (pVnode) vmReleaseVnode(pMgmt, pVnode); return -1; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 963bfa3197..973c45eda7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -112,6 +112,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { pVnode->diskPrimary = pCfg->diskPrimary; pVnode->refCount = 0; pVnode->dropped = 0; + pVnode->failed = 0; pVnode->path = taosStrdup(pCfg->path); pVnode->pImpl = pImpl; @@ -121,11 +122,15 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { return -1; } - if (vmAllocQueue(pMgmt, pVnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pVnode->path); - taosMemoryFree(pVnode); - return -1; + if (pImpl) { + if (vmAllocQueue(pMgmt, pVnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pVnode->path); + taosMemoryFree(pVnode); + return -1; + } + } else { + pVnode->failed = 1; } taosThreadRwlockWrlock(&pMgmt->lock); @@ -271,8 +276,10 @@ static void *vmOpenVnodeInThread(void *param) { if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d since %s", pCfg->vgId, pThread->threadIndex, terrstr()); - pThread->failed++; - continue; + if (terrno != TSDB_CODE_VND_NOT_EXIST) { + pThread->failed++; + continue; + } } if (vmOpenVnode(pMgmt, pCfg, pImpl) != 0) { @@ -379,6 +386,7 @@ static void *vmCloseVnodeInThread(void *param) { for (int32_t v = 0; v < pThread->vnodeNum; ++v) { SVnodeObj *pVnode = pThread->ppVnodes[v]; + if (pVnode->failed) continue; char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to close, %d of %d have been closed", pVnode->vgId, @@ -473,7 +481,9 @@ static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { if (ppVnodes != NULL) { for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = ppVnodes[i]; - vnodeSyncCheckTimeout(pVnode->pImpl); + if (!pVnode->failed) { + vnodeSyncCheckTimeout(pVnode->pImpl); + } vmReleaseVnode(pMgmt, pVnode); } taosMemoryFree(ppVnodes); @@ -605,6 +615,12 @@ static void *vmRestoreVnodeInThread(void *param) { for (int32_t v = 0; v < pThread->vnodeNum; ++v) { SVnodeObj *pVnode = pThread->ppVnodes[v]; + if (pVnode->failed) { + dError("vgId:%d, skip restoring vnode in failure mode.", pVnode->vgId); + continue; + } + + ASSERT(pVnode->pImpl); char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been restored", pVnode->vgId, diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 696107ca90..4b18ec4fb0 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -187,9 +187,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp pHead->vgId = ntohl(pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); - if (pVnode == NULL) { - dGWarn("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, - terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); + if (pVnode == NULL || pVnode->failed) { + dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); terrno = (terrno != 0) ? terrno : -1; return terrno; } @@ -316,7 +316,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { int32_t size = -1; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); - if (pVnode != NULL) { + if (pVnode != NULL && !pVnode->failed) { switch (qtype) { case WRITE_QUEUE: size = taosQueueItemSize(pVnode->pWriteW.queue); @@ -339,8 +339,8 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { default: break; } - vmReleaseVnode(pMgmt, pVnode); } + if (pVnode) vmReleaseVnode(pMgmt, pVnode); if (size < 0) { dTrace("vgId:%d, can't get size from queue since %s, qtype:%d", vgId, terrstr(), qtype); size = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 28d1e171d8..f9499cda6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -329,6 +329,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC char dir[TSDB_FILENAME_LEN] = {0}; char tdir[TSDB_FILENAME_LEN * 2] = {0}; int32_t ret = 0; + terrno = TSDB_CODE_SUCCESS; if (vnodeCheckDisk(diskPrimary, pTfs)) { vError("failed to open vnode from %s since %s. diskPrimary:%d", path, terrstr(), diskPrimary); @@ -342,6 +343,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC ret = vnodeLoadInfo(dir, &info); if (ret < 0) { vError("failed to open vnode from %s since %s", path, tstrerror(terrno)); + terrno = TSDB_CODE_VND_NOT_EXIST; return NULL; } @@ -514,7 +516,10 @@ void vnodeClose(SVnode *pVnode) { } // start the sync timer after the queue is ready -int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); } +int32_t vnodeStart(SVnode *pVnode) { + ASSERT(pVnode); + return vnodeSyncStart(pVnode); +} int32_t vnodeIsCatchUp(SVnode *pVnode) { return syncIsCatchUp(pVnode->sync); } From a515f8a94f576c2b1ec7f85df2c39e137417ba43 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 18 Oct 2023 14:40:00 +0800 Subject: [PATCH 138/177] feat: use vnode config info if existing during vnodeCreate --- source/dnode/vnode/src/vnd/vnodeOpen.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index f9499cda6d..7ba542cbf1 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -64,6 +64,13 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs info.state.applied = -1; info.state.commitID = 0; + SVnodeInfo oldInfo = {0}; + oldInfo.config = vnodeCfgDefault; + if (vnodeLoadInfo(dir, &oldInfo) == 0) { + vWarn("vgId:%d, vnode config info already exists at %s.", oldInfo.config.vgId, dir); + return (oldInfo.config.dbId == info.config.dbId) ? 0 : -1; + } + vInfo("vgId:%d, save config while create", info.config.vgId); if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir) < 0) { vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); From a2e0480839c932ae45d12509c4519575c224255f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 18 Oct 2023 16:09:51 +0800 Subject: [PATCH 139/177] refact: rename as SYNC_FSM_STATE_COMPLETE --- include/libs/sync/sync.h | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncMain.c | 2 +- source/libs/sync/src/syncPipeline.c | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 71c56e8c86..ad525a2aa7 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -87,7 +87,7 @@ typedef enum { } ESyncRole; typedef enum { - SYNC_FSM_STATE_NORMAL = 0, + SYNC_FSM_STATE_COMPLETE = 0, SYNC_FSM_STATE_INCOMPLETE, } ESyncFsmState; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 86fb7cb55b..6c03ed68e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -791,7 +791,7 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { pSnap->lastApplyIndex = pVnode->state.committed; pSnap->lastApplyTerm = pVnode->state.commitTerm; pSnap->lastConfigIndex = -1; - pSnap->state = SYNC_FSM_STATE_NORMAL; + pSnap->state = SYNC_FSM_STATE_COMPLETE; if (tsdbSnapGetFsState(pVnode) != TSDB_FS_STATE_NORMAL) { pSnap->state = SYNC_FSM_STATE_INCOMPLETE; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 647b86bae9..51a0679889 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -155,7 +155,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pEntry->term); - if (ths->fsmState != SYNC_FSM_STATE_NORMAL) { + if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) { pReply->fsmState = ths->fsmState; sWarn("vgId:%d, unable to accept, due to incomplete fsm state. index:%" PRId64, ths->vgId, pEntry->index); syncEntryDestroy(pEntry); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index eaaccecf90..f9dc10da02 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1010,7 +1010,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { sNTrace(pSyncNode, "reset commit index by snapshot"); } pSyncNode->fsmState = snapshot.state; - if (pSyncNode->fsmState != SYNC_FSM_STATE_NORMAL) { + if (pSyncNode->fsmState == SYNC_FSM_STATE_INCOMPLETE) { sError("vgId:%d, fsm state is incomplete.", pSyncNode->vgId); if (pSyncNode->replicaNum == 1) { goto _error; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index a38d67a388..a7ee37cc3b 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -840,8 +840,8 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn } if (pMsg->fsmState == SYNC_FSM_STATE_INCOMPLETE || (!pMsg->success && pMsg->matchIndex >= pMsg->lastSendIndex)) { - char* msg1 = "rollback match index failure"; - char* msg2 = "incomplete fsm state"; + char* msg1 = " rollback match index failure"; + char* msg2 = " incomplete fsm state"; sInfo("vgId:%d, snapshot replication to dnode:%d. reason:%s, match index:%" PRId64 ", last sent:%" PRId64, pNode->vgId, DID(&destId), (pMsg->fsmState == SYNC_FSM_STATE_INCOMPLETE ? msg2 : msg1), pMsg->matchIndex, pMsg->lastSendIndex); From 27b2d37bde83733575b844ce77e24a2eb159cbe2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 11:27:31 +0800 Subject: [PATCH 140/177] add rpc sync read timeout --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7edd6d7d63..ffa8e6d7da 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -160,7 +160,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp, 1000); + rpcSendRecvWithTimeout(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp, 2000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[256]; From b4b742b3fb611718cb83288d8248f6e7284fe20a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 11:35:29 +0800 Subject: [PATCH 141/177] add rpc sync read timeout --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ffa8e6d7da..3ca782d670 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -160,7 +160,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp, 2000); + rpcSendRecvWithTimeout(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp, 5000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[256]; From afac84465678104668812a72fa8efbca59f10e0e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 11:45:59 +0800 Subject: [PATCH 142/177] add rpc sync read time --- source/libs/transport/src/transCli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 842f961141..ef60c8a94e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2427,6 +2427,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { memcpy(pSyncMsg->pRsp, (char*)pResp, sizeof(*pResp)); tsem_post(pSyncMsg->pSem); taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); + } else { + rpcFreeCont(pResp->pCont); } } } else { From 0188289308ed4c77e3242f423aea8ea96da0fb45 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 14:07:54 +0800 Subject: [PATCH 143/177] statusClientRpc --- include/common/tmsgcb.h | 1 - source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 1 - source/dnode/mgmt/node_mgmt/src/dmTransport.c | 21 +++++++++---------- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 7ab69759e3..311bffb7da 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -52,7 +52,6 @@ typedef struct { void* data; void* mgmt; void* clientRpc; - void* statusClientRpc; void* serverRpc; PutToQueueFp putToQueueFp; GetQueueSizeFp qsizeFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3ca782d670..991f17f326 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -160,7 +160,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.statusClientRpc, &epSet, &rpcMsg, &rpcRsp, 5000); + rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, 5000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[256]; diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index f11378bb71..20789772e5 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -48,7 +48,6 @@ typedef struct { typedef struct { void *serverRpc; void *clientRpc; - void *statusClientRpc; SDnodeHandle msgHandles[TDMT_MAX]; } SDnodeTrans; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index dbd68a2fbe..ce6b21dd56 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -393,11 +393,11 @@ int32_t dmInitStatusClient(SDnode *pDnode) { rpcInit.timeToGetConn = tsTimeToGetAvailableConn; taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); - pTrans->statusClientRpc = rpcOpen(&rpcInit); - if (pTrans->statusClientRpc == NULL) { - dError("failed to init dnode rpc status client"); - return -1; - } + // pTrans->statusClientRpc = rpcOpen(&rpcInit); + // if (pTrans->statusClientRpc == NULL) { + // dError("failed to init dnode rpc status client"); + // return -1; + // } dDebug("dnode rpc status client is initialized"); return 0; @@ -413,11 +413,11 @@ void dmCleanupClient(SDnode *pDnode) { } void dmCleanupStatusClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; - if (pTrans->statusClientRpc) { - rpcClose(pTrans->statusClientRpc); - pTrans->statusClientRpc = NULL; - dDebug("dnode rpc status client is closed"); - } + // if (pTrans->statusClientRpc) { + // rpcClose(pTrans->statusClientRpc); + // pTrans->statusClientRpc = NULL; + // dDebug("dnode rpc status client is closed"); + // } } int32_t dmInitServer(SDnode *pDnode) { @@ -457,7 +457,6 @@ void dmCleanupServer(SDnode *pDnode) { SMsgCb dmGetMsgcb(SDnode *pDnode) { SMsgCb msgCb = { .clientRpc = pDnode->trans.clientRpc, - .statusClientRpc = pDnode->trans.statusClientRpc, .serverRpc = pDnode->trans.serverRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, From 312da0ac03557239bb77f94d251881e3b153546a Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 19 Oct 2023 13:59:41 +0800 Subject: [PATCH 144/177] docs(driver-go): update tmq auto.offset.reset configuration --- docs/en/07-develop/07-tmq.mdx | 4 +- docs/en/14-reference/03-connector/05-go.mdx | 88 ++++++++++++--------- docs/examples/go/sub/main.go | 34 ++++---- docs/zh/07-develop/07-tmq.md | 4 +- docs/zh/08-connector/20-go.mdx | 88 ++++++++++++--------- 5 files changed, 122 insertions(+), 96 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 47d82c0833..7dd06c9ca2 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -422,7 +422,7 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go conf := &tmq.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -511,7 +511,7 @@ var cfg = new ConsumerConfig GourpId = "TDengine-TMQ-C#", TDConnectUser = "root", TDConnectPasswd = "taosdata", - AutoOffsetReset = "earliest" + AutoOffsetReset = "latest" MsgWithTableName = "true", TDConnectIp = "127.0.0.1", TDConnectPort = "6030" diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index b3d4857d75..a0be7a4a02 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -794,7 +794,7 @@ The TDengine Go Connector supports subscription functionality with the following ```go consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -870,6 +870,7 @@ package main import ( "fmt" "os" + "time" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" @@ -890,19 +891,16 @@ func main() { if err != nil { panic(err) } - if err != nil { - panic(err) - } consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "earliest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_client", - "enable.auto.commit": "false", - "msg.with.table.name": "true", + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", }) if err != nil { panic(err) @@ -915,10 +913,16 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } + go func() { + for { + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) + } + }() + for i := 0; i < 5; i++ { ev := consumer.Poll(500) if ev != nil { @@ -972,6 +976,7 @@ package main import ( "database/sql" "fmt" + "time" "github.com/taosdata/driver-go/v3/common" tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" @@ -995,7 +1000,7 @@ func main() { "td.connect.pass": "taosdata", "group.id": "example", "client.id": "example_consumer", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", }) if err != nil { panic(err) @@ -1004,29 +1009,34 @@ func main() { if err != nil { panic(err) } + + _, err = db.Exec("create table example_ws_tmq.t_all(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") + if err != nil { + panic(err) + } go func() { - _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," + - "c1 bool," + - "c2 tinyint," + - "c3 smallint," + - "c4 int," + - "c5 bigint," + - "c6 tinyint unsigned," + - "c7 smallint unsigned," + - "c8 int unsigned," + - "c9 bigint unsigned," + - "c10 float," + - "c11 double," + - "c12 binary(20)," + - "c13 nchar(20)" + - ")") - if err != nil { - panic(err) - } - _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") - if err != nil { - panic(err) + for { + _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) } + }() for i := 0; i < 5; i++ { ev := consumer.Poll(500) diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index ed335cfdea..41cb33e94d 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "time" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" @@ -27,15 +28,15 @@ func main() { panic(err) } consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "earliest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_client", - "enable.auto.commit": "false", - "msg.with.table.name": "true", + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", }) if err != nil { panic(err) @@ -48,12 +49,17 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } + go func() { + for { + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + time.Sleep(time.Microsecond * 100) + } + }() for i := 0; i < 5; i++ { - ev := consumer.Poll(0) + ev := consumer.Poll(500) if ev != nil { switch e := ev.(type) { case *tmqcommon.DataMessage: diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 3272edbbca..a542c844fe 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -421,7 +421,7 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go conf := &tmq.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -512,7 +512,7 @@ var cfg = new ConsumerConfig GourpId = "TDengine-TMQ-C#", TDConnectUser = "root", TDConnectPasswd = "taosdata", - AutoOffsetReset = "earliest" + AutoOffsetReset = "latest" MsgWithTableName = "true", TDConnectIp = "127.0.0.1", TDConnectPort = "6030" diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 90ef4d83ca..3994278eef 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -797,7 +797,7 @@ TDengine Go 连接器支持订阅功能，应用 API 如下： ```go consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -873,6 +873,7 @@ package main import ( "fmt" "os" + "time" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" @@ -893,19 +894,16 @@ func main() { if err != nil { panic(err) } - if err != nil { - panic(err) - } consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "earliest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_client", - "enable.auto.commit": "false", - "msg.with.table.name": "true", + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", }) if err != nil { panic(err) @@ -918,10 +916,16 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } + go func() { + for { + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) + } + }() + for i := 0; i < 5; i++ { ev := consumer.Poll(500) if ev != nil { @@ -975,6 +979,7 @@ package main import ( "database/sql" "fmt" + "time" "github.com/taosdata/driver-go/v3/common" tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" @@ -998,7 +1003,7 @@ func main() { "td.connect.pass": "taosdata", "group.id": "example", "client.id": "example_consumer", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", }) if err != nil { panic(err) @@ -1007,29 +1012,34 @@ func main() { if err != nil { panic(err) } + + _, err = db.Exec("create table example_ws_tmq.t_all(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") + if err != nil { + panic(err) + } go func() { - _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," + - "c1 bool," + - "c2 tinyint," + - "c3 smallint," + - "c4 int," + - "c5 bigint," + - "c6 tinyint unsigned," + - "c7 smallint unsigned," + - "c8 int unsigned," + - "c9 bigint unsigned," + - "c10 float," + - "c11 double," + - "c12 binary(20)," + - "c13 nchar(20)" + - ")") - if err != nil { - panic(err) - } - _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") - if err != nil { - panic(err) + for { + _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) } + }() for i := 0; i < 5; i++ { ev := consumer.Poll(500) From 691bbff845052a99a960ad42d57ab2ec1acf8c8c Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Thu, 19 Oct 2023 14:06:39 +0800 Subject: [PATCH 145/177] docs(zh): update offset.reset default setting --- docs/zh/08-connector/26-rust.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 018552117e..37b1787707 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -447,7 +447,7 @@ consumer.unsubscribe().await; - `group.id`: 同一个消费者组，将以至少消费一次的方式进行消息负载均衡。 - `client.id`: 可选的订阅客户端识别项。 -- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认为从头订阅。注意，此选项在同一个 `group.id` 中仅生效一次。 +- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认值根据 TDengine 版本有所不同，详细参见 [数据订阅](https://docs.taosdata.com/develop/tmq/)。注意，此选项在同一个 `group.id` 中仅生效一次。 - `enable.auto.commit`: 当设置为 `true` 时，将启用自动标记模式，当对数据一致性不敏感时，可以启用此方式。 - `auto.commit.interval.ms`: 自动标记的时间间隔。 From 2297beea5c054e68bfc154ab3111163843a284ed Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Thu, 19 Oct 2023 14:06:48 +0800 Subject: [PATCH 146/177] docs(en): update offset.reset default setting --- docs/en/14-reference/03-connector/06-rust.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index a98683d43c..5a44b161cb 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -442,7 +442,7 @@ The following parameters can be configured for the TMQ DSN. Only `group.id` is m - `group.id`: Within a consumer group, load balancing is implemented by consuming messages on an at-least-once basis. - `client.id`: Subscriber client ID. -- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default is earliest. Note: This parameter is set per consumer group. +- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default value varies depending on the TDengine version. For details, see [Data Subscription](https://docs.tdengine.com/develop/tmq/). Note: This parameter is set per consumer group. - `enable.auto.commit`: Automatically commits. This can be enabled when data consistency is not essential. - `auto.commit.interval.ms`: Interval for automatic commits. From 93cac3532e0521dc5125589139551d3b6c143046 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 15:30:04 +0800 Subject: [PATCH 147/177] fix invalid debug info --- source/libs/transport/src/trans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 6842d9ee82..b23d229931 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -42,7 +42,7 @@ void* rpcOpen(const SRpcInit* pInit) { } if (pInit->label) { int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label); - tstrncpy(pRpc->label, pInit->label, len); + memcpy(pRpc->label, pInit->label, len); } pRpc->compressSize = pInit->compressSize; From 04c60486e919331aeca236908ec0ea6a59dbf18a Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 11 Oct 2023 14:08:42 +0800 Subject: [PATCH 148/177] feat: support interval syntax of quoted duration and default unit --- source/common/src/ttime.c | 3 + source/libs/parser/inc/sql.y | 17 +- source/libs/parser/src/parAstCreater.c | 35 +- source/libs/parser/src/sql.c | 4436 ++++++++++---------- source/libs/planner/src/planSpliter.c | 5 +- tests/parallel_test/cases.task | 4 + tests/system-test/2-query/interval_unit.py | 197 + 7 files changed, 2480 insertions(+), 2217 deletions(-) create mode 100644 tests/system-test/2-query/interval_unit.py diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 425218f0e1..723298f256 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -665,6 +665,9 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati if (*unit == 'n' || *unit == 'y') { return 0; } + if(isdigit(*unit)) { + *unit = getPrecisionUnit(timePrecision); + } return getDuration(*duration, *unit, duration, timePrecision); } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 2bab513602..6e92818bbb 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1119,19 +1119,24 @@ partition_item(A) ::= expr_or_subquery(B) column_alias(C). partition_item(A) ::= expr_or_subquery(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } twindow_clause_opt(A) ::= . { A = NULL; } -twindow_clause_opt(A) ::= - SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } +twindow_clause_opt(A) ::= SESSION NK_LP column_reference(B) NK_COMMA + interval_sliding_duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } twindow_clause_opt(A) ::= STATE_WINDOW NK_LP expr_or_subquery(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } +twindow_clause_opt(A) ::= INTERVAL NK_LP interval_sliding_duration_literal(B) + NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } -twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + INTERVAL NK_LP interval_sliding_duration_literal(B) NK_COMMA + interval_sliding_duration_literal(C) NK_RP sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); } twindow_clause_opt(A) ::= EVENT_WINDOW START WITH search_condition(B) END WITH search_condition(C). { A = createEventWindowNode(pCxt, B, C); } sliding_opt(A) ::= . { A = NULL; } -sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } +sliding_opt(A) ::= SLIDING NK_LP interval_sliding_duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } + +interval_sliding_duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +interval_sliding_duration_literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +interval_sliding_duration_literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 12062e0d4a..9966347219 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -499,7 +499,40 @@ SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) CHECK_PARSER_STATUS(pCxt); SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); CHECK_OUT_OF_MEM(val); - val->literal = strndup(pLiteral->z, pLiteral->n); + if (pLiteral->type == TK_NK_STRING) { + // like '100s' or "100d" + // check format: ^[0-9]+[smwbauhdny]$' + if (pLiteral->n < 4) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pLiteral->z); + return NULL; + } + char unit = pLiteral->z[pLiteral->n - 2]; + switch (unit) { + case 'a': + case 'b': + case 'd': + case 'h': + case 'm': + case 's': + case 'u': + case 'w': + case 'y': + case 'n': + break; + default: + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pLiteral->z); + return NULL; + } + for (uint32_t i = 1; i < pLiteral->n - 2; ++i) { + if (!isdigit(pLiteral->z[i])) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pLiteral->z); + return NULL; + } + } + val->literal = strndup(pLiteral->z + 1, pLiteral->n - 2); + } else { + val->literal = strndup(pLiteral->z, pLiteral->n); + } CHECK_OUT_OF_MEM(val->literal); val->isDuration = true; val->translate = false; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index f79ec34f03..8a8ea0bee5 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -449,29 +449,29 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 502 +#define YYNOCODE 503 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int32_t yy88; - EJoinType yy184; - SAlterOption yy233; - SToken yy269; - bool yy345; - SNode* yy348; - EShowKind yy361; - int8_t yy371; - int64_t yy537; - EOperatorType yy696; - SDataType yy720; - SShowTablesOption yy749; - EFillMode yy758; - ENullOrder yy841; - SNodeList* yy860; - EOrder yy870; - STokenPair yy993; + SNodeList* yy20; + int32_t yy114; + STokenPair yy157; + SNode* yy342; + EOperatorType yy356; + EJoinType yy392; + SToken yy479; + int8_t yy541; + EShowKind yy545; + EFillMode yy590; + EOrder yy592; + int64_t yy669; + ENullOrder yy689; + SShowTablesOption yy711; + SDataType yy750; + bool yy829; + SAlterOption yy857; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -488,17 +488,17 @@ typedef union { #define ParseCTX_STORE #define YYFALLBACK 1 #define YYNSTATE 820 -#define YYNRULE 624 -#define YYNRULE_WITH_ACTION 624 +#define YYNRULE 627 +#define YYNRULE_WITH_ACTION 627 #define YYNTOKEN 344 #define YY_MAX_SHIFT 819 -#define YY_MIN_SHIFTREDUCE 1211 -#define YY_MAX_SHIFTREDUCE 1834 -#define YY_ERROR_ACTION 1835 -#define YY_ACCEPT_ACTION 1836 -#define YY_NO_ACTION 1837 -#define YY_MIN_REDUCE 1838 -#define YY_MAX_REDUCE 2461 +#define YY_MIN_SHIFTREDUCE 1214 +#define YY_MAX_SHIFTREDUCE 1840 +#define YY_ERROR_ACTION 1841 +#define YY_ACCEPT_ACTION 1842 +#define YY_NO_ACTION 1843 +#define YY_MIN_REDUCE 1844 +#define YY_MAX_REDUCE 2470 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -567,652 +567,652 @@ typedef union { *********** Begin parsing tables **********************************************/ #define YY_ACTTAB_COUNT (3215) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 403, 707, 2024, 412, 674, 2217, 2437, 2432, 167, 2432, - /* 10 */ 2199, 164, 48, 46, 1761, 463, 2026, 668, 391, 2026, - /* 20 */ 409, 133, 1606, 1839, 673, 189, 2075, 2436, 590, 2433, - /* 30 */ 675, 2433, 2435, 418, 417, 1687, 1921, 1604, 47, 45, - /* 40 */ 44, 43, 42, 1758, 123, 2286, 2268, 122, 121, 120, - /* 50 */ 119, 118, 117, 116, 115, 114, 686, 142, 1613, 689, - /* 60 */ 515, 513, 260, 361, 1682, 1838, 1631, 203, 548, 706, - /* 70 */ 19, 549, 1881, 1836, 2246, 123, 2371, 1612, 122, 121, - /* 80 */ 120, 119, 118, 117, 116, 115, 114, 2286, 2015, 132, - /* 90 */ 131, 130, 129, 128, 127, 126, 125, 124, 2250, 2236, - /* 100 */ 1632, 723, 2368, 816, 667, 706, 15, 553, 791, 790, - /* 110 */ 789, 788, 421, 550, 787, 786, 146, 781, 780, 779, + /* 0 */ 2019, 707, 2030, 674, 2015, 2446, 2441, 548, 2441, 169, + /* 10 */ 549, 1887, 48, 46, 1764, 463, 2446, 1970, 412, 2441, + /* 20 */ 409, 133, 1609, 1845, 673, 193, 164, 2445, 590, 2442, + /* 30 */ 675, 2442, 2444, 391, 2032, 1690, 1927, 1607, 2445, 38, + /* 40 */ 313, 2081, 2442, 2443, 123, 91, 2274, 122, 121, 120, + /* 50 */ 119, 118, 117, 116, 115, 114, 686, 142, 1634, 689, + /* 60 */ 515, 513, 379, 361, 1685, 1844, 553, 207, 706, 1634, + /* 70 */ 19, 2025, 550, 1842, 2252, 123, 706, 1615, 122, 121, + /* 80 */ 120, 119, 118, 117, 116, 115, 114, 2292, 2021, 132, + /* 90 */ 131, 130, 129, 128, 127, 126, 125, 124, 2256, 2242, + /* 100 */ 706, 723, 1279, 816, 1278, 167, 15, 1867, 791, 790, + /* 110 */ 789, 788, 421, 2033, 787, 786, 146, 781, 780, 779, /* 120 */ 778, 777, 776, 775, 158, 771, 770, 769, 420, 419, - /* 130 */ 766, 765, 764, 177, 176, 763, 1631, 182, 2267, 648, - /* 140 */ 2252, 2303, 1689, 1690, 110, 2269, 727, 2271, 2272, 722, - /* 150 */ 717, 717, 424, 707, 2024, 51, 186, 423, 2356, 377, - /* 160 */ 2436, 2135, 405, 2352, 292, 2364, 685, 62, 134, 684, - /* 170 */ 563, 2432, 2155, 133, 1662, 1672, 2437, 191, 706, 2432, - /* 180 */ 595, 1688, 1691, 185, 67, 2386, 1765, 402, 673, 189, - /* 190 */ 2152, 694, 1631, 2433, 675, 2064, 1607, 2436, 1605, 625, - /* 200 */ 649, 2433, 2434, 2432, 556, 41, 40, 549, 1881, 47, - /* 210 */ 45, 44, 43, 42, 623, 1632, 621, 255, 254, 1616, - /* 220 */ 2438, 189, 1530, 1531, 1697, 2433, 675, 196, 1610, 1611, - /* 230 */ 1631, 1661, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, - /* 240 */ 719, 715, 1680, 1681, 1683, 1684, 1685, 1686, 2, 48, - /* 250 */ 46, 2155, 169, 1631, 360, 2268, 1629, 409, 2437, 1606, - /* 260 */ 1964, 1529, 1532, 500, 51, 60, 518, 369, 724, 2153, - /* 270 */ 694, 517, 1687, 646, 1604, 1633, 41, 40, 2077, 182, - /* 280 */ 47, 45, 44, 43, 42, 390, 62, 483, 1634, 519, - /* 290 */ 1510, 1511, 1251, 2075, 485, 414, 2286, 2077, 2070, 2072, - /* 300 */ 456, 1682, 1411, 2136, 399, 455, 1757, 19, 2236, 224, - /* 310 */ 723, 1716, 2075, 551, 1612, 1888, 1402, 752, 751, 750, - /* 320 */ 1406, 749, 1408, 1409, 748, 745, 296, 1417, 742, 1419, - /* 330 */ 1420, 739, 736, 733, 1253, 1256, 1257, 1446, 1447, 62, - /* 340 */ 816, 378, 1276, 15, 1275, 2002, 1663, 2267, 649, 1861, - /* 350 */ 2303, 2432, 471, 110, 2269, 727, 2271, 2272, 722, 138, - /* 360 */ 717, 567, 1824, 145, 1860, 151, 2327, 2356, 2438, 189, - /* 370 */ 1717, 405, 2352, 2433, 675, 469, 2131, 1277, 147, 1689, - /* 380 */ 1690, 52, 2142, 2121, 1663, 507, 506, 505, 504, 499, + /* 130 */ 766, 765, 764, 177, 176, 763, 567, 1280, 2273, 1634, + /* 140 */ 2258, 2309, 1692, 1693, 110, 2275, 727, 2277, 2278, 722, + /* 150 */ 717, 717, 424, 1279, 51, 1278, 190, 423, 2362, 67, + /* 160 */ 2242, 2161, 405, 2358, 293, 2370, 685, 62, 134, 684, + /* 170 */ 138, 2441, 707, 2030, 1665, 1675, 402, 195, 2205, 2158, + /* 180 */ 694, 1691, 1694, 1635, 2006, 2392, 51, 228, 1280, 673, + /* 190 */ 193, 551, 133, 1894, 2442, 675, 1610, 594, 1608, 595, + /* 200 */ 649, 593, 509, 2441, 556, 41, 40, 549, 1887, 47, + /* 210 */ 45, 44, 43, 42, 760, 156, 155, 757, 756, 755, + /* 220 */ 153, 2447, 193, 1513, 1514, 62, 2442, 675, 1613, 1614, + /* 230 */ 264, 1664, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, + /* 240 */ 719, 715, 1683, 1684, 1686, 1687, 1688, 1689, 2, 48, + /* 250 */ 46, 2161, 1635, 762, 360, 2274, 1632, 409, 2446, 1609, + /* 260 */ 170, 62, 1856, 500, 218, 52, 518, 369, 724, 2159, + /* 270 */ 694, 517, 1690, 186, 1607, 1636, 41, 40, 707, 2030, + /* 280 */ 47, 45, 44, 43, 42, 2252, 508, 483, 1637, 519, + /* 290 */ 469, 2137, 545, 1914, 485, 377, 2292, 2141, 198, 2261, + /* 300 */ 543, 1685, 1414, 539, 535, 1533, 1534, 19, 2242, 2256, + /* 310 */ 723, 1719, 564, 297, 1615, 610, 1405, 752, 751, 750, + /* 320 */ 1409, 749, 1411, 1412, 748, 745, 297, 1420, 742, 1422, + /* 330 */ 1423, 739, 736, 733, 47, 45, 44, 43, 42, 209, + /* 340 */ 816, 378, 230, 15, 1532, 1535, 551, 2273, 1894, 664, + /* 350 */ 2309, 2258, 471, 110, 2275, 727, 2277, 2278, 722, 1721, + /* 360 */ 717, 717, 1830, 145, 714, 151, 2333, 2362, 565, 2154, + /* 370 */ 1720, 405, 2358, 414, 1449, 1450, 2076, 2078, 186, 1692, + /* 380 */ 1693, 563, 2148, 2127, 297, 507, 506, 505, 504, 499, /* 390 */ 498, 497, 496, 495, 491, 490, 489, 488, 359, 480, - /* 400 */ 479, 478, 2236, 473, 472, 376, 2371, 1498, 1499, 664, - /* 410 */ 226, 1662, 1672, 1517, 551, 1366, 1888, 2236, 1688, 1691, - /* 420 */ 259, 41, 40, 296, 205, 47, 45, 44, 43, 42, - /* 430 */ 1365, 294, 2367, 1607, 1924, 1605, 760, 156, 155, 757, - /* 440 */ 756, 755, 153, 677, 272, 296, 30, 707, 2024, 37, - /* 450 */ 407, 1711, 1712, 1713, 1714, 1715, 1719, 1720, 1721, 1722, - /* 460 */ 170, 564, 1850, 686, 142, 1610, 1611, 194, 1661, 1664, - /* 470 */ 1665, 1666, 1667, 1668, 1669, 1670, 1671, 719, 715, 1680, - /* 480 */ 1681, 1683, 1684, 1685, 1686, 2, 12, 48, 46, 707, - /* 490 */ 2024, 2268, 669, 1575, 1576, 409, 520, 1606, 296, 594, - /* 500 */ 670, 665, 658, 593, 689, 1633, 609, 608, 607, 56, - /* 510 */ 1687, 1859, 1604, 599, 139, 603, 2077, 565, 2148, 602, - /* 520 */ 1801, 91, 1631, 375, 601, 606, 385, 384, 14, 13, - /* 530 */ 600, 2075, 2286, 596, 686, 142, 707, 2024, 379, 1682, - /* 540 */ 296, 609, 608, 607, 2236, 19, 723, 2019, 599, 139, - /* 550 */ 603, 12, 1612, 10, 602, 167, 460, 707, 2024, 601, - /* 560 */ 606, 385, 384, 2027, 2236, 600, 475, 2131, 596, 762, - /* 570 */ 688, 187, 2364, 2365, 634, 140, 2369, 461, 816, 38, - /* 580 */ 313, 15, 2268, 2267, 258, 1606, 2303, 12, 257, 110, - /* 590 */ 2269, 727, 2271, 2272, 722, 724, 717, 1890, 1259, 674, - /* 600 */ 1604, 186, 2432, 2356, 1630, 41, 40, 405, 2352, 47, - /* 610 */ 45, 44, 43, 42, 108, 208, 2108, 1689, 1690, 673, - /* 620 */ 189, 707, 2024, 2286, 2433, 675, 1858, 2229, 707, 2024, - /* 630 */ 2387, 143, 1908, 1857, 1730, 2236, 1856, 723, 2077, 2016, - /* 640 */ 1612, 477, 188, 2364, 2365, 404, 140, 2369, 492, 1662, - /* 650 */ 1672, 545, 94, 2075, 610, 364, 1688, 1691, 389, 543, - /* 660 */ 627, 294, 539, 535, 383, 382, 816, 2001, 707, 2024, - /* 670 */ 1276, 1607, 1275, 1605, 2267, 649, 1663, 2303, 2432, 2236, - /* 680 */ 110, 2269, 727, 2271, 2272, 722, 2236, 717, 493, 2236, - /* 690 */ 502, 2131, 2452, 1355, 2356, 2438, 189, 1612, 405, 2352, - /* 700 */ 2433, 675, 1789, 1610, 1611, 1277, 1661, 1664, 1665, 1666, - /* 710 */ 1667, 1668, 1669, 1670, 1671, 719, 715, 1680, 1681, 1683, - /* 720 */ 1684, 1685, 1686, 2, 48, 46, 1692, 2268, 707, 2024, - /* 730 */ 2077, 2077, 409, 1357, 1606, 381, 380, 413, 592, 213, - /* 740 */ 724, 1999, 2394, 707, 2024, 2075, 693, 1687, 566, 1604, - /* 750 */ 661, 660, 1787, 1788, 1790, 1791, 1792, 686, 142, 1607, - /* 760 */ 594, 1605, 753, 2021, 593, 41, 40, 2371, 2286, 47, - /* 770 */ 45, 44, 43, 42, 707, 2024, 1682, 487, 707, 2024, - /* 780 */ 2236, 1370, 723, 304, 305, 1831, 486, 1855, 303, 1612, - /* 790 */ 773, 1610, 1611, 2366, 261, 91, 1369, 2268, 269, 44, - /* 800 */ 43, 42, 760, 156, 155, 757, 756, 755, 153, 102, - /* 810 */ 724, 709, 2407, 2328, 62, 816, 707, 2024, 49, 2267, - /* 820 */ 2268, 2020, 2303, 2071, 2072, 110, 2269, 727, 2271, 2272, - /* 830 */ 722, 444, 717, 724, 2017, 656, 692, 2452, 2286, 2356, - /* 840 */ 2236, 41, 40, 405, 2352, 47, 45, 44, 43, 42, - /* 850 */ 2236, 711, 723, 2328, 1689, 1690, 1279, 1280, 446, 442, - /* 860 */ 2000, 2286, 522, 1854, 1800, 190, 2364, 2365, 2013, 140, - /* 870 */ 2369, 586, 585, 2236, 319, 723, 760, 156, 155, 757, - /* 880 */ 756, 755, 153, 210, 707, 2024, 1662, 1672, 1830, 2267, - /* 890 */ 588, 587, 2303, 1688, 1691, 110, 2269, 727, 2271, 2272, - /* 900 */ 722, 2009, 717, 454, 308, 453, 1634, 2452, 1607, 2356, - /* 910 */ 1605, 1899, 2267, 405, 2352, 2303, 2236, 2011, 110, 2269, - /* 920 */ 727, 2271, 2272, 722, 87, 717, 412, 86, 1853, 762, - /* 930 */ 2452, 2230, 2356, 612, 167, 452, 405, 2352, 1852, 509, - /* 940 */ 1610, 1611, 2026, 1661, 1664, 1665, 1666, 1667, 1668, 1669, - /* 950 */ 1670, 1671, 719, 715, 1680, 1681, 1683, 1684, 1685, 1686, - /* 960 */ 2, 48, 46, 2246, 2007, 2268, 1849, 605, 604, 409, - /* 970 */ 2077, 1606, 2028, 296, 785, 783, 1634, 2255, 724, 649, - /* 980 */ 2425, 2236, 2432, 614, 1687, 702, 1604, 2250, 9, 41, - /* 990 */ 40, 2236, 1848, 47, 45, 44, 43, 42, 626, 2438, - /* 1000 */ 189, 214, 707, 2024, 2433, 675, 2286, 1718, 144, 166, - /* 1010 */ 678, 2327, 754, 1682, 256, 2068, 707, 2024, 2236, 2236, - /* 1020 */ 723, 34, 704, 508, 707, 2024, 1612, 41, 40, 2252, - /* 1030 */ 617, 47, 45, 44, 43, 42, 705, 611, 415, 717, - /* 1040 */ 1847, 707, 2024, 253, 314, 2236, 167, 632, 758, 265, - /* 1050 */ 1846, 2068, 816, 718, 2026, 49, 2268, 2267, 681, 2246, - /* 1060 */ 2303, 416, 206, 110, 2269, 727, 2271, 2272, 722, 724, - /* 1070 */ 717, 2375, 714, 2254, 1845, 2452, 1844, 2356, 1866, 811, - /* 1080 */ 1851, 405, 2352, 2250, 71, 2220, 35, 70, 1777, 1843, - /* 1090 */ 1842, 1689, 1690, 2236, 36, 649, 1723, 2286, 2432, 1841, - /* 1100 */ 41, 40, 690, 2236, 47, 45, 44, 43, 42, 2236, - /* 1110 */ 629, 723, 628, 2077, 759, 2438, 189, 2068, 268, 328, - /* 1120 */ 2433, 675, 2054, 1662, 1672, 2252, 406, 2236, 2076, 2236, - /* 1130 */ 1688, 1691, 271, 41, 40, 717, 431, 47, 45, 44, - /* 1140 */ 43, 42, 2236, 2236, 154, 1607, 774, 1605, 2267, 1986, - /* 1150 */ 649, 2303, 2236, 2432, 110, 2269, 727, 2271, 2272, 722, - /* 1160 */ 154, 717, 1256, 1257, 2376, 1750, 2452, 3, 2356, 2118, - /* 1170 */ 2438, 189, 405, 2352, 75, 2433, 675, 1610, 1611, 54, - /* 1180 */ 1661, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 719, - /* 1190 */ 715, 1680, 1681, 1683, 1684, 1685, 1686, 2, 48, 46, - /* 1200 */ 135, 597, 2268, 598, 246, 154, 409, 244, 1606, 1897, - /* 1210 */ 1750, 467, 84, 248, 250, 724, 247, 249, 1570, 252, - /* 1220 */ 270, 1687, 251, 1604, 85, 1352, 2257, 1350, 1833, 1834, - /* 1230 */ 1708, 615, 50, 1310, 55, 2268, 50, 276, 1615, 154, - /* 1240 */ 1965, 14, 13, 2286, 107, 1614, 50, 301, 721, 72, - /* 1250 */ 1682, 152, 154, 104, 2400, 2236, 289, 723, 65, 662, - /* 1260 */ 137, 283, 767, 1612, 95, 679, 1962, 768, 1961, 50, - /* 1270 */ 50, 731, 1891, 1311, 392, 152, 2286, 154, 422, 1573, - /* 1280 */ 136, 152, 2287, 2259, 2140, 1882, 1329, 2390, 2236, 816, - /* 1290 */ 723, 1327, 15, 1887, 2267, 2065, 687, 2303, 291, 5, - /* 1300 */ 110, 2269, 727, 2271, 2272, 722, 1786, 717, 288, 425, - /* 1310 */ 1785, 278, 2331, 691, 2356, 682, 430, 373, 405, 2352, - /* 1320 */ 1527, 306, 295, 699, 809, 310, 1396, 2267, 1689, 1690, - /* 1330 */ 2303, 438, 1724, 350, 2269, 727, 2271, 2272, 722, 720, - /* 1340 */ 717, 708, 2321, 1673, 327, 1424, 439, 168, 1637, 1428, - /* 1350 */ 448, 1435, 335, 447, 1433, 157, 197, 198, 450, 200, - /* 1360 */ 1662, 1672, 1551, 322, 1629, 464, 1630, 1688, 1691, 332, - /* 1370 */ 74, 468, 212, 73, 470, 1634, 2141, 474, 476, 481, - /* 1380 */ 511, 494, 1607, 357, 1605, 503, 501, 2133, 510, 512, - /* 1390 */ 523, 217, 524, 521, 222, 530, 528, 525, 216, 1618, - /* 1400 */ 526, 219, 529, 527, 531, 1635, 1617, 546, 4, 547, - /* 1410 */ 555, 554, 227, 557, 1610, 1611, 1632, 1661, 1664, 1665, - /* 1420 */ 1666, 1667, 1668, 1669, 1670, 1671, 719, 715, 1680, 1681, - /* 1430 */ 1683, 1684, 1685, 1686, 2, 62, 418, 417, 558, 229, - /* 1440 */ 1636, 559, 1638, 560, 232, 562, 1620, 234, 1639, 89, - /* 1450 */ 2149, 90, 568, 589, 591, 239, 2014, 243, 2010, 1687, - /* 1460 */ 245, 1613, 160, 161, 618, 2012, 2008, 112, 619, 354, - /* 1470 */ 162, 2268, 631, 63, 633, 93, 163, 637, 262, 636, - /* 1480 */ 323, 148, 638, 264, 724, 266, 642, 1558, 1682, 2208, - /* 1490 */ 2205, 2204, 644, 8, 2406, 663, 641, 643, 653, 659, - /* 1500 */ 697, 1612, 395, 2405, 672, 284, 666, 2268, 2378, 2391, - /* 1510 */ 2401, 652, 2286, 282, 654, 285, 396, 82, 81, 459, - /* 1520 */ 724, 274, 202, 175, 2236, 277, 723, 713, 651, 1633, - /* 1530 */ 286, 2455, 683, 287, 2431, 451, 449, 680, 2268, 141, - /* 1540 */ 1750, 1755, 290, 1753, 2372, 179, 358, 297, 2286, 440, - /* 1550 */ 324, 724, 437, 433, 429, 426, 452, 695, 149, 696, - /* 1560 */ 2236, 2163, 723, 2267, 2162, 2161, 2303, 325, 700, 110, - /* 1570 */ 2269, 727, 2271, 2272, 722, 401, 717, 701, 150, 2286, - /* 1580 */ 61, 2329, 326, 2356, 2337, 103, 729, 405, 2352, 101, - /* 1590 */ 1, 2236, 2025, 723, 296, 192, 317, 2069, 329, 2267, - /* 1600 */ 1987, 1235, 2303, 2268, 813, 110, 2269, 727, 2271, 2272, - /* 1610 */ 722, 810, 717, 159, 815, 338, 724, 710, 331, 2356, - /* 1620 */ 1621, 353, 1616, 405, 2352, 333, 2228, 53, 352, 2227, - /* 1630 */ 2267, 2226, 79, 2303, 365, 342, 111, 2269, 727, 2271, - /* 1640 */ 2272, 722, 2221, 717, 2286, 427, 1597, 366, 428, 1598, - /* 1650 */ 2356, 195, 1624, 1626, 2355, 2352, 2236, 2219, 723, 432, - /* 1660 */ 434, 435, 436, 2218, 1596, 715, 1680, 1681, 1683, 1684, - /* 1670 */ 1685, 1686, 374, 2216, 441, 2215, 443, 2268, 2214, 1586, - /* 1680 */ 445, 2195, 199, 2194, 201, 1554, 80, 1553, 2176, 2175, - /* 1690 */ 724, 2174, 457, 2173, 458, 2267, 2172, 2123, 2303, 462, - /* 1700 */ 2268, 111, 2269, 727, 2271, 2272, 722, 1497, 717, 2117, - /* 1710 */ 466, 465, 2114, 724, 204, 2356, 83, 2112, 2286, 712, - /* 1720 */ 2352, 2113, 2111, 2116, 2115, 207, 2110, 2109, 2107, 2106, - /* 1730 */ 2236, 209, 723, 2105, 482, 2104, 484, 2120, 2103, 2102, - /* 1740 */ 2101, 2286, 2100, 2099, 2098, 2097, 2096, 2095, 2094, 2093, - /* 1750 */ 2092, 2091, 2090, 2236, 2089, 723, 211, 2088, 88, 2087, - /* 1760 */ 2086, 2085, 2119, 2084, 2083, 215, 2081, 514, 2268, 725, - /* 1770 */ 2082, 2080, 2303, 516, 2079, 111, 2269, 727, 2271, 2272, - /* 1780 */ 722, 724, 717, 2268, 2078, 1927, 362, 1503, 1926, 2356, - /* 1790 */ 1367, 1371, 2267, 368, 2352, 2303, 724, 1925, 171, 2269, - /* 1800 */ 727, 2271, 2272, 722, 1363, 717, 363, 1923, 1920, 2286, - /* 1810 */ 532, 1919, 1912, 536, 534, 1901, 540, 218, 544, 220, - /* 1820 */ 1877, 2236, 221, 723, 2286, 538, 533, 223, 241, 537, - /* 1830 */ 1258, 541, 542, 77, 183, 2256, 2236, 1876, 723, 2193, - /* 1840 */ 650, 2397, 225, 2183, 174, 184, 78, 552, 2171, 2268, - /* 1850 */ 2170, 231, 584, 580, 576, 572, 233, 240, 2147, 236, - /* 1860 */ 2267, 2003, 724, 2303, 1922, 1918, 172, 2269, 727, 2271, - /* 1870 */ 2272, 722, 569, 717, 1303, 2267, 570, 1916, 2303, 573, - /* 1880 */ 571, 111, 2269, 727, 2271, 2272, 722, 574, 717, 575, - /* 1890 */ 2286, 1914, 577, 579, 1911, 2356, 578, 581, 92, 583, - /* 1900 */ 2353, 238, 2236, 582, 723, 1896, 1894, 1895, 1893, 1873, - /* 1910 */ 2005, 1440, 242, 2004, 2268, 1439, 64, 1354, 1353, 676, - /* 1920 */ 2453, 1351, 1349, 1348, 1347, 1346, 1345, 724, 782, 2268, - /* 1930 */ 784, 1909, 1900, 1342, 1340, 1341, 1339, 1898, 386, 387, - /* 1940 */ 388, 2267, 724, 613, 2303, 1872, 1871, 171, 2269, 727, - /* 1950 */ 2271, 2272, 722, 1870, 717, 2286, 616, 620, 1869, 622, - /* 1960 */ 393, 1868, 624, 113, 1580, 1582, 1579, 2236, 29, 723, - /* 1970 */ 2286, 237, 230, 1584, 2192, 394, 68, 1562, 235, 561, - /* 1980 */ 1564, 57, 2236, 2182, 723, 2268, 1560, 639, 165, 640, - /* 1990 */ 2398, 2169, 2168, 267, 1539, 1538, 2437, 228, 724, 20, - /* 2000 */ 645, 6, 17, 281, 647, 21, 2267, 31, 273, 2303, - /* 2010 */ 1803, 275, 351, 2269, 727, 2271, 2272, 722, 655, 717, - /* 2020 */ 7, 2267, 22, 657, 2303, 2257, 2286, 351, 2269, 727, - /* 2030 */ 2271, 2272, 722, 1784, 717, 280, 173, 33, 2236, 279, - /* 2040 */ 723, 32, 66, 1776, 96, 24, 1823, 1818, 2268, 1824, - /* 2050 */ 1817, 397, 1822, 1821, 398, 293, 59, 23, 58, 1747, - /* 2060 */ 178, 724, 18, 2268, 1746, 2167, 2146, 98, 97, 25, - /* 2070 */ 2145, 300, 698, 99, 1782, 302, 724, 2267, 2268, 307, - /* 2080 */ 2303, 69, 312, 344, 2269, 727, 2271, 2272, 722, 2286, - /* 2090 */ 717, 721, 104, 100, 26, 13, 11, 1622, 1699, 1698, - /* 2100 */ 1709, 2236, 309, 723, 2286, 2306, 180, 1677, 1675, 400, - /* 2110 */ 1674, 716, 193, 39, 16, 27, 2236, 1654, 723, 2286, - /* 2120 */ 730, 411, 734, 737, 740, 1646, 28, 732, 671, 1425, - /* 2130 */ 726, 2236, 735, 723, 1422, 728, 1421, 1418, 738, 743, - /* 2140 */ 2267, 2268, 741, 2303, 1412, 1410, 172, 2269, 727, 2271, - /* 2150 */ 2272, 722, 744, 717, 724, 2267, 746, 747, 2303, 105, - /* 2160 */ 315, 351, 2269, 727, 2271, 2272, 722, 1416, 717, 1415, - /* 2170 */ 2267, 2268, 1414, 2303, 1413, 106, 350, 2269, 727, 2271, - /* 2180 */ 2272, 722, 2286, 717, 724, 2322, 1434, 408, 76, 1430, - /* 2190 */ 1336, 1301, 761, 1333, 2236, 1332, 723, 1331, 635, 1330, - /* 2200 */ 2454, 1328, 1326, 1325, 772, 1324, 1361, 316, 1322, 1321, - /* 2210 */ 1320, 1319, 2286, 1318, 1317, 1316, 819, 410, 1358, 1356, - /* 2220 */ 1313, 1312, 1307, 1309, 2236, 1308, 723, 1306, 1917, 792, - /* 2230 */ 793, 1915, 321, 2267, 1913, 794, 2303, 798, 796, 351, - /* 2240 */ 2269, 727, 2271, 2272, 722, 797, 717, 800, 181, 801, - /* 2250 */ 802, 1910, 804, 805, 806, 1892, 807, 803, 799, 795, - /* 2260 */ 808, 318, 2268, 2267, 1248, 1867, 2303, 1236, 812, 351, - /* 2270 */ 2269, 727, 2271, 2272, 722, 724, 717, 818, 320, 814, - /* 2280 */ 1837, 1608, 330, 817, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2290 */ 1837, 1837, 1837, 1837, 1837, 2268, 1837, 1837, 1837, 1837, - /* 2300 */ 1837, 1837, 109, 2286, 1837, 311, 1837, 1837, 724, 1837, - /* 2310 */ 1837, 1837, 1837, 1837, 1837, 2236, 1837, 723, 2268, 1837, - /* 2320 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2330 */ 1837, 724, 1837, 2268, 1837, 1837, 2286, 703, 1837, 1837, - /* 2340 */ 1837, 1837, 1837, 1837, 1837, 1837, 724, 1837, 2236, 1837, - /* 2350 */ 723, 1837, 1837, 1837, 630, 1837, 1837, 2303, 1837, 2286, - /* 2360 */ 346, 2269, 727, 2271, 2272, 722, 1837, 717, 1837, 1837, - /* 2370 */ 1837, 2236, 1837, 723, 2286, 299, 1837, 1837, 1837, 1837, - /* 2380 */ 1837, 1837, 298, 1837, 1837, 1837, 2236, 2267, 723, 1837, - /* 2390 */ 2303, 1837, 1837, 336, 2269, 727, 2271, 2272, 722, 1837, - /* 2400 */ 717, 263, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2268, - /* 2410 */ 2267, 1837, 1837, 2303, 1837, 1837, 334, 2269, 727, 2271, - /* 2420 */ 2272, 722, 724, 717, 2268, 2267, 1837, 1837, 2303, 1837, - /* 2430 */ 1837, 337, 2269, 727, 2271, 2272, 722, 724, 717, 2268, - /* 2440 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2450 */ 2286, 1837, 724, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2460 */ 1837, 1837, 2236, 1837, 723, 2286, 1837, 1837, 1837, 1837, - /* 2470 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2236, 1837, 723, - /* 2480 */ 2286, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2490 */ 1837, 1837, 2236, 1837, 723, 1837, 1837, 1837, 1837, 1837, - /* 2500 */ 1837, 2267, 2268, 1837, 2303, 1837, 1837, 343, 2269, 727, - /* 2510 */ 2271, 2272, 722, 1837, 717, 724, 2267, 1837, 1837, 2303, - /* 2520 */ 1837, 1837, 347, 2269, 727, 2271, 2272, 722, 1837, 717, - /* 2530 */ 1837, 2267, 2268, 1837, 2303, 1837, 1837, 339, 2269, 727, - /* 2540 */ 2271, 2272, 722, 2286, 717, 724, 1837, 1837, 1837, 1837, - /* 2550 */ 1837, 1837, 1837, 1837, 1837, 2236, 1837, 723, 1837, 1837, - /* 2560 */ 1837, 1837, 1837, 1837, 1837, 2268, 1837, 1837, 1837, 1837, - /* 2570 */ 1837, 1837, 1837, 2286, 1837, 1837, 1837, 1837, 724, 1837, - /* 2580 */ 1837, 1837, 1837, 1837, 1837, 2236, 1837, 723, 1837, 1837, - /* 2590 */ 1837, 1837, 1837, 1837, 2267, 1837, 1837, 2303, 1837, 1837, - /* 2600 */ 348, 2269, 727, 2271, 2272, 722, 2286, 717, 1837, 1837, - /* 2610 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2236, 1837, - /* 2620 */ 723, 1837, 1837, 1837, 2267, 1837, 1837, 2303, 1837, 1837, - /* 2630 */ 340, 2269, 727, 2271, 2272, 722, 1837, 717, 1837, 1837, - /* 2640 */ 1837, 1837, 1837, 2268, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2650 */ 1837, 1837, 1837, 1837, 1837, 1837, 724, 2267, 1837, 1837, - /* 2660 */ 2303, 1837, 1837, 349, 2269, 727, 2271, 2272, 722, 1837, - /* 2670 */ 717, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2268, - /* 2680 */ 1837, 1837, 1837, 1837, 2286, 1837, 1837, 1837, 1837, 1837, - /* 2690 */ 1837, 1837, 724, 1837, 1837, 1837, 2236, 1837, 723, 1837, - /* 2700 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2268, 1837, - /* 2710 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2720 */ 2286, 724, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2730 */ 1837, 1837, 2236, 1837, 723, 2267, 1837, 1837, 2303, 1837, - /* 2740 */ 1837, 341, 2269, 727, 2271, 2272, 722, 1837, 717, 2286, - /* 2750 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2760 */ 1837, 2236, 1837, 723, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2770 */ 1837, 2267, 1837, 2268, 2303, 1837, 1837, 355, 2269, 727, - /* 2780 */ 2271, 2272, 722, 1837, 717, 1837, 724, 1837, 1837, 1837, - /* 2790 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2800 */ 2267, 1837, 1837, 2303, 1837, 1837, 356, 2269, 727, 2271, - /* 2810 */ 2272, 722, 1837, 717, 2286, 1837, 1837, 1837, 1837, 1837, - /* 2820 */ 1837, 1837, 1837, 1837, 1837, 1837, 2236, 1837, 723, 1837, - /* 2830 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2840 */ 2268, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2850 */ 1837, 1837, 1837, 724, 1837, 1837, 2268, 1837, 1837, 1837, - /* 2860 */ 1837, 1837, 1837, 1837, 1837, 2267, 1837, 1837, 2303, 724, - /* 2870 */ 1837, 2280, 2269, 727, 2271, 2272, 722, 1837, 717, 1837, - /* 2880 */ 1837, 2286, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2890 */ 1837, 1837, 1837, 2236, 1837, 723, 1837, 2286, 1837, 1837, - /* 2900 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2236, - /* 2910 */ 1837, 723, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2920 */ 1837, 2268, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2930 */ 1837, 1837, 2267, 1837, 724, 2303, 1837, 1837, 2279, 2269, - /* 2940 */ 727, 2271, 2272, 722, 1837, 717, 1837, 1837, 2267, 1837, - /* 2950 */ 2268, 2303, 1837, 1837, 2278, 2269, 727, 2271, 2272, 722, - /* 2960 */ 1837, 717, 2286, 724, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2970 */ 1837, 1837, 1837, 1837, 2236, 1837, 723, 1837, 1837, 1837, - /* 2980 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 2990 */ 1837, 2286, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 3000 */ 1837, 1837, 1837, 2236, 1837, 723, 1837, 1837, 1837, 1837, - /* 3010 */ 1837, 1837, 1837, 2267, 1837, 2268, 2303, 1837, 1837, 370, - /* 3020 */ 2269, 727, 2271, 2272, 722, 1837, 717, 1837, 724, 1837, - /* 3030 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 3040 */ 1837, 1837, 2267, 1837, 2268, 2303, 1837, 1837, 371, 2269, - /* 3050 */ 727, 2271, 2272, 722, 1837, 717, 2286, 724, 1837, 1837, - /* 3060 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2236, 1837, - /* 3070 */ 723, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 3080 */ 2268, 1837, 1837, 1837, 1837, 2286, 1837, 1837, 1837, 1837, - /* 3090 */ 1837, 1837, 1837, 724, 1837, 1837, 1837, 2236, 1837, 723, - /* 3100 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 2267, 1837, 2268, - /* 3110 */ 2303, 1837, 1837, 367, 2269, 727, 2271, 2272, 722, 1837, - /* 3120 */ 717, 2286, 724, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 3130 */ 1837, 1837, 1837, 2236, 1837, 723, 2267, 1837, 1837, 2303, - /* 3140 */ 1837, 1837, 372, 2269, 727, 2271, 2272, 722, 1837, 717, - /* 3150 */ 2286, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 3160 */ 1837, 1837, 2236, 1837, 723, 1837, 1837, 1837, 1837, 1837, - /* 3170 */ 1837, 1837, 725, 1837, 1837, 2303, 1837, 1837, 346, 2269, - /* 3180 */ 727, 2271, 2272, 722, 1837, 717, 1837, 1837, 1837, 1837, - /* 3190 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 3200 */ 1837, 2267, 1837, 1837, 2303, 1837, 1837, 345, 2269, 727, - /* 3210 */ 2271, 2272, 722, 1837, 717, + /* 400 */ 479, 478, 2142, 473, 472, 376, 2377, 1501, 1502, 1768, + /* 410 */ 2226, 1665, 1675, 1520, 62, 1634, 707, 2030, 1691, 1694, + /* 420 */ 297, 41, 40, 2252, 297, 47, 45, 44, 43, 42, + /* 430 */ 107, 295, 2374, 1610, 1930, 1608, 56, 2260, 35, 104, + /* 440 */ 670, 665, 658, 1866, 276, 625, 30, 2256, 1726, 37, + /* 450 */ 407, 1714, 1715, 1716, 1717, 1718, 1722, 1723, 1724, 1725, + /* 460 */ 623, 431, 621, 259, 258, 1613, 1614, 2124, 1664, 1667, + /* 470 */ 1668, 1669, 1670, 1671, 1672, 1673, 1674, 719, 715, 1683, + /* 480 */ 1684, 1686, 1687, 1688, 1689, 2, 12, 48, 46, 2258, + /* 490 */ 406, 2274, 418, 417, 1254, 409, 2242, 1609, 1636, 717, + /* 500 */ 1865, 707, 2030, 454, 689, 453, 609, 608, 607, 467, + /* 510 */ 1690, 1864, 1607, 599, 139, 603, 2083, 1616, 669, 602, + /* 520 */ 1807, 460, 1711, 375, 601, 606, 385, 384, 1700, 762, + /* 530 */ 600, 2081, 2292, 596, 1634, 452, 1256, 1259, 1260, 1685, + /* 540 */ 1358, 609, 608, 607, 2242, 19, 723, 444, 599, 139, + /* 550 */ 603, 668, 1615, 2242, 602, 686, 142, 2077, 2078, 601, + /* 560 */ 606, 385, 384, 1863, 2242, 600, 475, 2137, 596, 1666, + /* 570 */ 2083, 456, 1609, 297, 446, 442, 455, 390, 816, 2292, + /* 580 */ 1360, 15, 2274, 2273, 262, 2081, 2309, 1607, 261, 110, + /* 590 */ 2275, 727, 2277, 2278, 722, 724, 717, 1896, 12, 674, + /* 600 */ 10, 190, 2441, 2362, 634, 41, 40, 405, 2358, 47, + /* 610 */ 45, 44, 43, 42, 12, 212, 2242, 1692, 1693, 649, + /* 620 */ 673, 193, 2441, 2292, 1787, 2442, 675, 1615, 707, 2030, + /* 630 */ 2393, 707, 2030, 14, 13, 2242, 102, 723, 667, 1788, + /* 640 */ 2447, 193, 502, 2137, 2083, 2442, 675, 1862, 461, 1665, + /* 650 */ 1675, 477, 94, 816, 295, 364, 1691, 1694, 389, 693, + /* 660 */ 627, 2023, 688, 191, 2370, 2371, 2007, 140, 2375, 707, + /* 670 */ 2030, 1610, 144, 1608, 2273, 2333, 1861, 2309, 1619, 1786, + /* 680 */ 110, 2275, 727, 2277, 2278, 722, 2083, 717, 1666, 492, + /* 690 */ 2083, 217, 2461, 399, 2362, 1282, 1283, 404, 405, 2358, + /* 700 */ 2242, 2081, 1795, 1613, 1614, 2081, 1664, 1667, 1668, 1669, + /* 710 */ 1670, 1671, 1672, 1673, 1674, 719, 715, 1683, 1684, 1686, + /* 720 */ 1687, 1688, 1689, 2, 48, 46, 1695, 2274, 403, 2242, + /* 730 */ 614, 189, 409, 648, 1609, 1615, 167, 707, 2030, 1733, + /* 740 */ 724, 2005, 2400, 2070, 2032, 626, 1610, 1690, 1608, 1607, + /* 750 */ 661, 660, 1793, 1794, 1796, 1797, 1798, 493, 41, 40, + /* 760 */ 91, 260, 47, 45, 44, 43, 42, 412, 2292, 773, + /* 770 */ 214, 44, 43, 42, 319, 167, 1685, 617, 1613, 1614, + /* 780 */ 2242, 34, 723, 2032, 611, 1837, 2026, 41, 40, 1615, + /* 790 */ 257, 47, 45, 44, 43, 42, 108, 2274, 2377, 586, + /* 800 */ 585, 760, 156, 155, 757, 756, 755, 153, 588, 587, + /* 810 */ 724, 87, 2413, 143, 86, 816, 707, 2030, 49, 2273, + /* 820 */ 2274, 2022, 2309, 753, 2373, 110, 2275, 727, 2277, 2278, + /* 830 */ 722, 71, 717, 724, 70, 656, 566, 2461, 2292, 2362, + /* 840 */ 1637, 41, 40, 405, 2358, 47, 45, 44, 43, 42, + /* 850 */ 2242, 2083, 723, 263, 1692, 1693, 1262, 1806, 413, 60, + /* 860 */ 2114, 2292, 1633, 1860, 41, 40, 2081, 646, 47, 45, + /* 870 */ 44, 43, 42, 2242, 2017, 723, 760, 156, 155, 757, + /* 880 */ 756, 755, 153, 707, 2030, 75, 1665, 1675, 1836, 2273, + /* 890 */ 1897, 1369, 2309, 1691, 1694, 110, 2275, 727, 2277, 2278, + /* 900 */ 722, 2013, 717, 2027, 605, 604, 1368, 2461, 1610, 2362, + /* 910 */ 1608, 1859, 2273, 405, 2358, 2309, 2242, 1858, 110, 2275, + /* 920 */ 727, 2277, 2278, 722, 1855, 717, 1578, 1579, 2235, 2445, + /* 930 */ 2461, 2034, 2362, 383, 382, 85, 405, 2358, 686, 142, + /* 940 */ 1613, 1614, 809, 1664, 1667, 1668, 1669, 1670, 1671, 1672, + /* 950 */ 1673, 1674, 719, 715, 1683, 1684, 1686, 1687, 1688, 1689, + /* 960 */ 2, 48, 46, 1780, 2242, 2274, 2236, 1761, 632, 409, + /* 970 */ 2242, 1609, 520, 2377, 415, 1854, 649, 2242, 724, 2441, + /* 980 */ 2434, 36, 167, 709, 1690, 2334, 1607, 41, 40, 9, + /* 990 */ 2032, 47, 45, 44, 43, 42, 210, 2447, 193, 2372, + /* 1000 */ 1853, 678, 2442, 675, 381, 380, 2292, 592, 1637, 166, + /* 1010 */ 690, 707, 2030, 1685, 649, 3, 649, 2441, 2242, 2441, + /* 1020 */ 723, 487, 785, 783, 686, 142, 1615, 54, 2242, 594, + /* 1030 */ 486, 265, 711, 593, 2334, 2447, 193, 2447, 193, 1373, + /* 1040 */ 2442, 675, 2442, 675, 304, 305, 192, 2370, 2371, 303, + /* 1050 */ 140, 2375, 816, 2242, 1372, 49, 2274, 2273, 649, 754, + /* 1060 */ 2309, 2441, 2074, 110, 2275, 727, 2277, 2278, 722, 724, + /* 1070 */ 717, 2381, 1852, 707, 2030, 2461, 1851, 2362, 1634, 2447, + /* 1080 */ 193, 405, 2358, 269, 2442, 675, 707, 2030, 2083, 2083, + /* 1090 */ 272, 1692, 1693, 273, 41, 40, 1850, 2292, 47, 45, + /* 1100 */ 44, 43, 42, 702, 2082, 135, 692, 707, 2030, 2242, + /* 1110 */ 1849, 723, 1848, 2008, 1847, 707, 2030, 84, 707, 2030, + /* 1120 */ 522, 707, 2030, 1665, 1675, 2242, 2223, 308, 597, 2242, + /* 1130 */ 1691, 1694, 194, 2370, 2371, 704, 140, 2375, 705, 1872, + /* 1140 */ 811, 314, 707, 2030, 154, 1610, 147, 1608, 2273, 2242, + /* 1150 */ 758, 2309, 1355, 2074, 110, 2275, 727, 2277, 2278, 722, + /* 1160 */ 154, 717, 416, 2242, 275, 2242, 2461, 2242, 2362, 1259, + /* 1170 */ 1260, 759, 405, 2358, 2074, 154, 328, 1613, 1614, 2060, + /* 1180 */ 1664, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 719, + /* 1190 */ 715, 1683, 1684, 1686, 1687, 1688, 1689, 2, 48, 46, + /* 1200 */ 774, 598, 2274, 1992, 2382, 1753, 409, 250, 1609, 1905, + /* 1210 */ 248, 1753, 677, 252, 254, 724, 251, 253, 55, 256, + /* 1220 */ 1903, 1690, 255, 1607, 629, 1353, 628, 274, 681, 2263, + /* 1230 */ 1760, 612, 1666, 1313, 1573, 2274, 50, 50, 1618, 178, + /* 1240 */ 767, 154, 615, 2292, 1839, 1840, 50, 302, 721, 1576, + /* 1250 */ 1685, 72, 152, 14, 13, 2242, 679, 723, 154, 718, + /* 1260 */ 1971, 1857, 2406, 1615, 1332, 290, 662, 768, 137, 65, + /* 1270 */ 50, 95, 1968, 1314, 284, 50, 2292, 731, 1967, 152, + /* 1280 */ 154, 136, 152, 659, 392, 395, 2265, 1617, 2242, 816, + /* 1290 */ 723, 1330, 15, 666, 2273, 2293, 696, 2309, 422, 2146, + /* 1300 */ 110, 2275, 727, 2277, 2278, 722, 1888, 717, 2071, 687, + /* 1310 */ 1792, 1791, 2337, 281, 2362, 691, 1893, 2396, 405, 2358, + /* 1320 */ 1530, 306, 292, 289, 296, 699, 310, 2273, 1692, 1693, + /* 1330 */ 2309, 5, 1399, 350, 2275, 727, 2277, 2278, 722, 720, + /* 1340 */ 717, 708, 2327, 1727, 1676, 425, 430, 168, 200, 327, + /* 1350 */ 438, 1427, 335, 1431, 1438, 1436, 157, 373, 439, 1640, + /* 1360 */ 1665, 1675, 448, 447, 202, 201, 450, 1691, 1694, 332, + /* 1370 */ 74, 204, 1554, 73, 1632, 464, 322, 1633, 468, 216, + /* 1380 */ 470, 1637, 1610, 357, 1608, 2147, 474, 476, 511, 481, + /* 1390 */ 494, 510, 501, 2139, 226, 530, 528, 525, 503, 1621, + /* 1400 */ 512, 523, 524, 521, 220, 221, 526, 527, 223, 529, + /* 1410 */ 531, 1638, 546, 4, 1613, 1614, 554, 1664, 1667, 1668, + /* 1420 */ 1669, 1670, 1671, 1672, 1673, 1674, 719, 715, 1683, 1684, + /* 1430 */ 1686, 1687, 1688, 1689, 2, 62, 418, 417, 547, 555, + /* 1440 */ 557, 231, 1635, 558, 233, 1639, 1623, 1641, 1620, 559, + /* 1450 */ 560, 562, 236, 238, 1642, 2155, 568, 89, 589, 1690, + /* 1460 */ 90, 1616, 243, 591, 618, 2020, 247, 2016, 249, 160, + /* 1470 */ 619, 2274, 631, 63, 2214, 161, 112, 354, 93, 2018, + /* 1480 */ 633, 2014, 148, 162, 724, 682, 323, 163, 1685, 266, + /* 1490 */ 637, 638, 636, 268, 270, 642, 2211, 644, 1561, 663, + /* 1500 */ 2210, 1615, 2412, 641, 697, 2411, 8, 2274, 672, 653, + /* 1510 */ 285, 283, 2292, 2384, 654, 652, 2397, 82, 81, 459, + /* 1520 */ 724, 2407, 206, 287, 2242, 286, 723, 713, 651, 1753, + /* 1530 */ 278, 643, 396, 141, 280, 451, 449, 175, 2274, 1636, + /* 1540 */ 2464, 1756, 288, 291, 683, 680, 358, 1758, 2292, 440, + /* 1550 */ 2440, 724, 437, 433, 429, 426, 452, 324, 182, 298, + /* 1560 */ 2242, 695, 723, 2273, 149, 2378, 2309, 2169, 2168, 110, + /* 1570 */ 2275, 727, 2277, 2278, 722, 2167, 717, 325, 700, 2292, + /* 1580 */ 701, 2335, 401, 2362, 326, 61, 103, 405, 2358, 2075, + /* 1590 */ 1993, 2242, 150, 723, 297, 2031, 1, 2343, 101, 2273, + /* 1600 */ 329, 1238, 2309, 2274, 196, 110, 2275, 727, 2277, 2278, + /* 1610 */ 722, 317, 717, 729, 810, 813, 724, 710, 353, 2362, + /* 1620 */ 1624, 159, 1619, 405, 2358, 815, 53, 333, 331, 365, + /* 1630 */ 2273, 2234, 2233, 2309, 2232, 366, 111, 2275, 727, 2277, + /* 1640 */ 2278, 722, 338, 717, 2292, 352, 342, 79, 2227, 427, + /* 1650 */ 2362, 428, 1627, 1629, 2361, 2358, 2242, 1600, 723, 1601, + /* 1660 */ 199, 432, 2225, 434, 435, 715, 1683, 1684, 1686, 1687, + /* 1670 */ 1688, 1689, 436, 1599, 2224, 374, 2222, 2274, 441, 2221, + /* 1680 */ 443, 2220, 445, 1589, 2201, 203, 2200, 205, 1557, 80, + /* 1690 */ 724, 1556, 2182, 2181, 2180, 2273, 457, 458, 2309, 2179, + /* 1700 */ 2274, 111, 2275, 727, 2277, 2278, 722, 2178, 717, 2129, + /* 1710 */ 462, 2123, 1500, 724, 465, 2362, 466, 2120, 2292, 712, + /* 1720 */ 2358, 208, 2119, 83, 2118, 2117, 2122, 2121, 2116, 211, + /* 1730 */ 2242, 213, 723, 2115, 2113, 2112, 2111, 482, 2110, 484, + /* 1740 */ 2126, 2292, 2109, 2108, 2107, 2106, 2105, 2104, 2103, 2102, + /* 1750 */ 2101, 2100, 2099, 2242, 2098, 723, 2097, 2096, 2095, 215, + /* 1760 */ 2094, 2093, 2092, 88, 2091, 2125, 2090, 2089, 2274, 725, + /* 1770 */ 2088, 219, 2309, 2087, 514, 111, 2275, 727, 2277, 2278, + /* 1780 */ 722, 724, 717, 2274, 2086, 1506, 2085, 516, 2084, 2362, + /* 1790 */ 1370, 362, 2273, 368, 2358, 2309, 724, 1374, 171, 2275, + /* 1800 */ 727, 2277, 2278, 722, 1933, 717, 1932, 1931, 1366, 2292, + /* 1810 */ 1929, 1926, 1925, 532, 534, 1918, 1907, 222, 536, 363, + /* 1820 */ 540, 2242, 1883, 723, 2292, 533, 544, 538, 245, 542, + /* 1830 */ 187, 537, 1261, 541, 77, 227, 2242, 1882, 723, 2199, + /* 1840 */ 650, 2403, 224, 225, 174, 2262, 229, 188, 552, 2274, + /* 1850 */ 78, 2189, 584, 580, 576, 572, 2177, 244, 237, 235, + /* 1860 */ 2273, 2176, 724, 2309, 2153, 2009, 172, 2275, 727, 2277, + /* 1870 */ 2278, 722, 240, 717, 1928, 2273, 1306, 1924, 2309, 569, + /* 1880 */ 570, 111, 2275, 727, 2277, 2278, 722, 571, 717, 1922, + /* 1890 */ 2292, 573, 574, 1920, 575, 2362, 577, 578, 92, 579, + /* 1900 */ 2359, 242, 2242, 1917, 723, 581, 582, 1902, 583, 1900, + /* 1910 */ 1901, 1899, 1879, 2011, 2274, 64, 1443, 1442, 2010, 1357, + /* 1920 */ 676, 2462, 1356, 1354, 1352, 246, 1343, 724, 1351, 2274, + /* 1930 */ 1915, 1350, 782, 1349, 1348, 784, 386, 1345, 1344, 1906, + /* 1940 */ 1904, 2273, 724, 1342, 2309, 387, 388, 171, 2275, 727, + /* 1950 */ 2277, 2278, 722, 616, 717, 2292, 1878, 1877, 613, 1876, + /* 1960 */ 393, 1875, 620, 622, 1874, 624, 113, 2242, 1583, 723, + /* 1970 */ 2292, 241, 234, 1585, 1582, 394, 1587, 29, 239, 561, + /* 1980 */ 2198, 68, 2242, 2188, 723, 2274, 1563, 1565, 639, 2175, + /* 1990 */ 2404, 2174, 645, 271, 1567, 57, 640, 232, 724, 165, + /* 2000 */ 2446, 1542, 1541, 20, 647, 1809, 2273, 17, 6, 2309, + /* 2010 */ 7, 21, 351, 2275, 727, 2277, 2278, 722, 22, 717, + /* 2020 */ 180, 2273, 657, 31, 2309, 277, 2292, 351, 2275, 727, + /* 2030 */ 2277, 2278, 722, 655, 717, 279, 33, 282, 2242, 1790, + /* 2040 */ 723, 173, 2263, 66, 179, 24, 1824, 32, 2274, 1779, + /* 2050 */ 96, 1823, 1829, 397, 1830, 1828, 1827, 398, 1750, 1749, + /* 2060 */ 294, 724, 59, 2274, 181, 23, 2173, 2152, 97, 98, + /* 2070 */ 301, 2151, 18, 25, 99, 1785, 724, 2273, 2274, 183, + /* 2080 */ 2309, 307, 69, 344, 2275, 727, 2277, 2278, 722, 2292, + /* 2090 */ 717, 721, 104, 58, 312, 13, 100, 26, 1702, 309, + /* 2100 */ 11, 2242, 698, 723, 2292, 1701, 1625, 1680, 1712, 400, + /* 2110 */ 2312, 716, 184, 1678, 39, 728, 2242, 1677, 723, 2292, + /* 2120 */ 1649, 197, 1657, 16, 730, 411, 734, 27, 28, 671, + /* 2130 */ 1428, 2242, 732, 723, 737, 1425, 735, 738, 726, 740, + /* 2140 */ 2273, 2274, 1424, 2309, 1421, 741, 172, 2275, 727, 2277, + /* 2150 */ 2278, 722, 743, 717, 724, 2273, 1415, 744, 2309, 1413, + /* 2160 */ 746, 351, 2275, 727, 2277, 2278, 722, 747, 717, 1419, + /* 2170 */ 2273, 2274, 315, 2309, 1418, 1417, 350, 2275, 727, 2277, + /* 2180 */ 2278, 722, 2292, 717, 724, 2328, 1416, 408, 105, 106, + /* 2190 */ 1437, 76, 1433, 1304, 2242, 761, 723, 1339, 635, 1336, + /* 2200 */ 1335, 2463, 1334, 1333, 1331, 1329, 1328, 1364, 1327, 772, + /* 2210 */ 316, 1325, 2292, 1324, 1322, 1323, 819, 410, 1321, 1320, + /* 2220 */ 1319, 1361, 1310, 1359, 2242, 1316, 723, 1315, 1312, 1923, + /* 2230 */ 1311, 1309, 321, 2273, 1921, 792, 2309, 794, 793, 351, + /* 2240 */ 2275, 727, 2277, 2278, 722, 796, 717, 797, 185, 798, + /* 2250 */ 1919, 800, 801, 802, 1916, 805, 807, 803, 799, 795, + /* 2260 */ 804, 318, 2274, 2273, 806, 1898, 2309, 808, 1251, 351, + /* 2270 */ 2275, 727, 2277, 2278, 722, 724, 717, 1873, 1239, 812, + /* 2280 */ 320, 814, 1843, 1611, 330, 817, 818, 1843, 1843, 1843, + /* 2290 */ 1843, 1843, 1843, 1843, 1843, 2274, 1843, 1843, 1843, 1843, + /* 2300 */ 1843, 1843, 109, 2292, 1843, 311, 1843, 1843, 724, 1843, + /* 2310 */ 1843, 1843, 1843, 1843, 1843, 2242, 1843, 723, 2274, 1843, + /* 2320 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2330 */ 1843, 724, 1843, 2274, 1843, 1843, 2292, 703, 1843, 1843, + /* 2340 */ 1843, 1843, 1843, 1843, 1843, 1843, 724, 1843, 2242, 1843, + /* 2350 */ 723, 1843, 1843, 1843, 630, 1843, 1843, 2309, 1843, 2292, + /* 2360 */ 346, 2275, 727, 2277, 2278, 722, 1843, 717, 1843, 1843, + /* 2370 */ 1843, 2242, 1843, 723, 2292, 300, 1843, 1843, 1843, 1843, + /* 2380 */ 1843, 1843, 299, 1843, 1843, 1843, 2242, 2273, 723, 1843, + /* 2390 */ 2309, 1843, 1843, 336, 2275, 727, 2277, 2278, 722, 1843, + /* 2400 */ 717, 267, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2274, + /* 2410 */ 2273, 1843, 1843, 2309, 1843, 1843, 334, 2275, 727, 2277, + /* 2420 */ 2278, 722, 724, 717, 2274, 2273, 1843, 1843, 2309, 1843, + /* 2430 */ 1843, 337, 2275, 727, 2277, 2278, 722, 724, 717, 2274, + /* 2440 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2450 */ 2292, 1843, 724, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2460 */ 1843, 1843, 2242, 1843, 723, 2292, 1843, 1843, 1843, 1843, + /* 2470 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2242, 1843, 723, + /* 2480 */ 2292, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2490 */ 1843, 1843, 2242, 1843, 723, 1843, 1843, 1843, 1843, 1843, + /* 2500 */ 1843, 2273, 2274, 1843, 2309, 1843, 1843, 343, 2275, 727, + /* 2510 */ 2277, 2278, 722, 1843, 717, 724, 2273, 1843, 1843, 2309, + /* 2520 */ 1843, 1843, 347, 2275, 727, 2277, 2278, 722, 1843, 717, + /* 2530 */ 1843, 2273, 2274, 1843, 2309, 1843, 1843, 339, 2275, 727, + /* 2540 */ 2277, 2278, 722, 2292, 717, 724, 1843, 1843, 1843, 1843, + /* 2550 */ 1843, 1843, 1843, 1843, 1843, 2242, 1843, 723, 1843, 1843, + /* 2560 */ 1843, 1843, 1843, 1843, 1843, 2274, 1843, 1843, 1843, 1843, + /* 2570 */ 1843, 1843, 1843, 2292, 1843, 1843, 1843, 1843, 724, 1843, + /* 2580 */ 1843, 1843, 1843, 1843, 1843, 2242, 1843, 723, 1843, 1843, + /* 2590 */ 1843, 1843, 1843, 1843, 2273, 1843, 1843, 2309, 1843, 1843, + /* 2600 */ 348, 2275, 727, 2277, 2278, 722, 2292, 717, 1843, 1843, + /* 2610 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2242, 1843, + /* 2620 */ 723, 1843, 1843, 1843, 2273, 1843, 1843, 2309, 1843, 1843, + /* 2630 */ 340, 2275, 727, 2277, 2278, 722, 1843, 717, 1843, 1843, + /* 2640 */ 1843, 1843, 1843, 2274, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2650 */ 1843, 1843, 1843, 1843, 1843, 1843, 724, 2273, 1843, 1843, + /* 2660 */ 2309, 1843, 1843, 349, 2275, 727, 2277, 2278, 722, 1843, + /* 2670 */ 717, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2274, + /* 2680 */ 1843, 1843, 1843, 1843, 2292, 1843, 1843, 1843, 1843, 1843, + /* 2690 */ 1843, 1843, 724, 1843, 1843, 1843, 2242, 1843, 723, 1843, + /* 2700 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2274, 1843, + /* 2710 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2720 */ 2292, 724, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2730 */ 1843, 1843, 2242, 1843, 723, 2273, 1843, 1843, 2309, 1843, + /* 2740 */ 1843, 341, 2275, 727, 2277, 2278, 722, 1843, 717, 2292, + /* 2750 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2760 */ 1843, 2242, 1843, 723, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2770 */ 1843, 2273, 1843, 2274, 2309, 1843, 1843, 355, 2275, 727, + /* 2780 */ 2277, 2278, 722, 1843, 717, 1843, 724, 1843, 1843, 1843, + /* 2790 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2800 */ 2273, 1843, 1843, 2309, 1843, 1843, 356, 2275, 727, 2277, + /* 2810 */ 2278, 722, 1843, 717, 2292, 1843, 1843, 1843, 1843, 1843, + /* 2820 */ 1843, 1843, 1843, 1843, 1843, 1843, 2242, 1843, 723, 1843, + /* 2830 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2840 */ 2274, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2850 */ 1843, 1843, 1843, 724, 1843, 1843, 2274, 1843, 1843, 1843, + /* 2860 */ 1843, 1843, 1843, 1843, 1843, 2273, 1843, 1843, 2309, 724, + /* 2870 */ 1843, 2286, 2275, 727, 2277, 2278, 722, 1843, 717, 1843, + /* 2880 */ 1843, 2292, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2890 */ 1843, 1843, 1843, 2242, 1843, 723, 1843, 2292, 1843, 1843, + /* 2900 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2242, + /* 2910 */ 1843, 723, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2920 */ 1843, 2274, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2930 */ 1843, 1843, 2273, 1843, 724, 2309, 1843, 1843, 2285, 2275, + /* 2940 */ 727, 2277, 2278, 722, 1843, 717, 1843, 1843, 2273, 1843, + /* 2950 */ 2274, 2309, 1843, 1843, 2284, 2275, 727, 2277, 2278, 722, + /* 2960 */ 1843, 717, 2292, 724, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2970 */ 1843, 1843, 1843, 1843, 2242, 1843, 723, 1843, 1843, 1843, + /* 2980 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 2990 */ 1843, 2292, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 3000 */ 1843, 1843, 1843, 2242, 1843, 723, 1843, 1843, 1843, 1843, + /* 3010 */ 1843, 1843, 1843, 2273, 1843, 2274, 2309, 1843, 1843, 370, + /* 3020 */ 2275, 727, 2277, 2278, 722, 1843, 717, 1843, 724, 1843, + /* 3030 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 3040 */ 1843, 1843, 2273, 1843, 2274, 2309, 1843, 1843, 371, 2275, + /* 3050 */ 727, 2277, 2278, 722, 1843, 717, 2292, 724, 1843, 1843, + /* 3060 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2242, 1843, + /* 3070 */ 723, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 3080 */ 2274, 1843, 1843, 1843, 1843, 2292, 1843, 1843, 1843, 1843, + /* 3090 */ 1843, 1843, 1843, 724, 1843, 1843, 1843, 2242, 1843, 723, + /* 3100 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 2273, 1843, 2274, + /* 3110 */ 2309, 1843, 1843, 367, 2275, 727, 2277, 2278, 722, 1843, + /* 3120 */ 717, 2292, 724, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 3130 */ 1843, 1843, 1843, 2242, 1843, 723, 2273, 1843, 1843, 2309, + /* 3140 */ 1843, 1843, 372, 2275, 727, 2277, 2278, 722, 1843, 717, + /* 3150 */ 2292, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 3160 */ 1843, 1843, 2242, 1843, 723, 1843, 1843, 1843, 1843, 1843, + /* 3170 */ 1843, 1843, 725, 1843, 1843, 2309, 1843, 1843, 346, 2275, + /* 3180 */ 727, 2277, 2278, 722, 1843, 717, 1843, 1843, 1843, 1843, + /* 3190 */ 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, 1843, + /* 3200 */ 1843, 2273, 1843, 1843, 2309, 1843, 1843, 345, 2275, 727, + /* 3210 */ 2277, 2278, 722, 1843, 717, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 380, 359, 360, 380, 471, 0, 471, 474, 388, 474, - /* 10 */ 384, 388, 12, 13, 14, 359, 396, 360, 395, 396, - /* 20 */ 20, 379, 22, 0, 491, 492, 403, 492, 386, 496, - /* 30 */ 497, 496, 497, 12, 13, 35, 0, 37, 12, 13, - /* 40 */ 14, 15, 16, 4, 21, 388, 347, 24, 25, 26, - /* 50 */ 27, 28, 29, 30, 31, 32, 359, 360, 37, 360, - /* 60 */ 404, 405, 436, 407, 64, 0, 20, 411, 354, 20, - /* 70 */ 70, 357, 358, 344, 376, 21, 444, 77, 24, 25, + /* 0 */ 389, 359, 360, 471, 389, 471, 474, 354, 474, 369, + /* 10 */ 357, 358, 12, 13, 14, 359, 471, 377, 380, 474, + /* 20 */ 20, 379, 22, 0, 492, 493, 388, 493, 386, 497, + /* 30 */ 498, 497, 498, 395, 396, 35, 0, 37, 493, 460, + /* 40 */ 461, 403, 497, 498, 21, 368, 347, 24, 25, 26, + /* 50 */ 27, 28, 29, 30, 31, 32, 359, 360, 20, 360, + /* 60 */ 404, 405, 385, 407, 64, 0, 14, 411, 20, 20, + /* 70 */ 70, 394, 20, 344, 376, 21, 20, 77, 24, 25, /* 80 */ 26, 27, 28, 29, 30, 31, 32, 388, 390, 24, /* 90 */ 25, 26, 27, 28, 29, 30, 31, 32, 400, 400, - /* 100 */ 20, 402, 470, 103, 447, 20, 106, 14, 72, 73, - /* 110 */ 74, 75, 76, 20, 78, 79, 80, 81, 82, 83, + /* 100 */ 20, 402, 20, 103, 22, 388, 106, 347, 72, 73, + /* 110 */ 74, 75, 76, 396, 78, 79, 80, 81, 82, 83, /* 120 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 130 */ 94, 95, 96, 97, 98, 99, 20, 388, 439, 50, + /* 130 */ 94, 95, 96, 97, 98, 99, 69, 55, 439, 20, /* 140 */ 442, 442, 142, 143, 445, 446, 447, 448, 449, 450, - /* 150 */ 452, 452, 423, 359, 360, 106, 457, 428, 459, 410, - /* 160 */ 3, 412, 463, 464, 467, 468, 469, 106, 471, 472, - /* 170 */ 20, 474, 402, 379, 174, 175, 471, 478, 20, 474, - /* 180 */ 386, 181, 182, 387, 4, 486, 14, 417, 491, 492, - /* 190 */ 420, 421, 20, 496, 497, 399, 196, 492, 198, 21, - /* 200 */ 471, 496, 497, 474, 354, 8, 9, 357, 358, 12, - /* 210 */ 13, 14, 15, 16, 36, 20, 38, 39, 40, 198, - /* 220 */ 491, 492, 142, 143, 14, 496, 497, 222, 228, 229, - /* 230 */ 20, 231, 232, 233, 234, 235, 236, 237, 238, 239, + /* 150 */ 452, 452, 423, 20, 106, 22, 457, 428, 459, 4, + /* 160 */ 400, 402, 463, 464, 467, 468, 469, 106, 471, 472, + /* 170 */ 37, 474, 359, 360, 174, 175, 417, 478, 384, 420, + /* 180 */ 421, 181, 182, 20, 0, 486, 106, 355, 55, 492, + /* 190 */ 493, 359, 379, 361, 497, 498, 196, 135, 198, 386, + /* 200 */ 471, 139, 86, 474, 354, 8, 9, 357, 358, 12, + /* 210 */ 13, 14, 15, 16, 135, 136, 137, 138, 139, 140, + /* 220 */ 141, 492, 493, 174, 175, 106, 497, 498, 228, 229, + /* 230 */ 436, 231, 232, 233, 234, 235, 236, 237, 238, 239, /* 240 */ 240, 241, 242, 243, 244, 245, 246, 247, 248, 12, - /* 250 */ 13, 402, 369, 20, 18, 347, 20, 20, 3, 22, - /* 260 */ 377, 181, 182, 27, 106, 176, 30, 70, 360, 420, - /* 270 */ 421, 35, 35, 184, 37, 20, 8, 9, 388, 388, - /* 280 */ 12, 13, 14, 15, 16, 395, 106, 51, 20, 53, - /* 290 */ 174, 175, 4, 403, 58, 398, 388, 388, 401, 402, - /* 300 */ 423, 64, 103, 412, 395, 428, 267, 70, 400, 355, - /* 310 */ 402, 114, 403, 359, 77, 361, 117, 118, 119, 120, + /* 250 */ 13, 402, 20, 69, 18, 347, 20, 20, 3, 22, + /* 260 */ 346, 106, 348, 27, 148, 106, 30, 70, 360, 420, + /* 270 */ 421, 35, 35, 388, 37, 20, 8, 9, 359, 360, + /* 280 */ 12, 13, 14, 15, 16, 376, 170, 51, 20, 53, + /* 290 */ 359, 360, 51, 0, 58, 410, 388, 412, 379, 390, + /* 300 */ 59, 64, 103, 62, 63, 142, 143, 70, 400, 400, + /* 310 */ 402, 114, 359, 265, 77, 22, 117, 118, 119, 120, /* 320 */ 121, 122, 123, 124, 125, 126, 265, 128, 129, 130, - /* 330 */ 131, 132, 133, 134, 46, 47, 48, 142, 143, 106, - /* 340 */ 103, 105, 20, 106, 22, 0, 174, 439, 471, 347, - /* 350 */ 442, 474, 116, 445, 446, 447, 448, 449, 450, 37, - /* 360 */ 452, 69, 107, 455, 347, 457, 458, 459, 491, 492, - /* 370 */ 173, 463, 464, 496, 497, 359, 360, 55, 33, 142, - /* 380 */ 143, 106, 146, 147, 174, 149, 150, 151, 152, 153, + /* 330 */ 131, 132, 133, 134, 12, 13, 14, 15, 16, 408, + /* 340 */ 103, 105, 355, 106, 181, 182, 359, 439, 361, 180, + /* 350 */ 442, 442, 116, 445, 446, 447, 448, 449, 450, 173, + /* 360 */ 452, 452, 107, 455, 70, 457, 458, 459, 415, 416, + /* 370 */ 173, 463, 464, 398, 142, 143, 401, 402, 388, 142, + /* 380 */ 143, 20, 146, 147, 265, 149, 150, 151, 152, 153, /* 390 */ 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, - /* 400 */ 164, 165, 400, 167, 168, 169, 444, 171, 172, 180, - /* 410 */ 355, 174, 175, 177, 359, 22, 361, 400, 181, 182, - /* 420 */ 136, 8, 9, 265, 408, 12, 13, 14, 15, 16, - /* 430 */ 37, 176, 470, 196, 0, 198, 135, 136, 137, 138, - /* 440 */ 139, 140, 141, 286, 176, 265, 33, 359, 360, 252, + /* 400 */ 164, 165, 412, 167, 168, 169, 444, 171, 172, 14, + /* 410 */ 0, 174, 175, 177, 106, 20, 359, 360, 181, 182, + /* 420 */ 265, 8, 9, 376, 265, 12, 13, 14, 15, 16, + /* 430 */ 106, 176, 470, 196, 0, 198, 379, 390, 252, 115, + /* 440 */ 271, 272, 273, 347, 176, 21, 33, 400, 262, 252, /* 450 */ 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - /* 460 */ 346, 359, 348, 359, 360, 228, 229, 379, 231, 232, + /* 460 */ 36, 51, 38, 39, 40, 228, 229, 0, 231, 232, /* 470 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, - /* 480 */ 243, 244, 245, 246, 247, 248, 249, 12, 13, 359, - /* 490 */ 360, 347, 20, 209, 210, 20, 103, 22, 265, 135, - /* 500 */ 271, 272, 273, 139, 360, 20, 72, 73, 74, 379, - /* 510 */ 35, 347, 37, 79, 80, 81, 388, 415, 416, 85, - /* 520 */ 107, 368, 20, 395, 90, 91, 92, 93, 1, 2, - /* 530 */ 96, 403, 388, 99, 359, 360, 359, 360, 385, 64, - /* 540 */ 265, 72, 73, 74, 400, 70, 402, 394, 79, 80, - /* 550 */ 81, 249, 77, 251, 85, 388, 379, 359, 360, 90, - /* 560 */ 91, 92, 93, 396, 400, 96, 359, 360, 99, 69, - /* 570 */ 466, 467, 468, 469, 116, 471, 472, 379, 103, 460, - /* 580 */ 461, 106, 347, 439, 137, 22, 442, 249, 141, 445, - /* 590 */ 446, 447, 448, 449, 450, 360, 452, 362, 14, 471, - /* 600 */ 37, 457, 474, 459, 20, 8, 9, 463, 464, 12, - /* 610 */ 13, 14, 15, 16, 366, 408, 0, 142, 143, 491, - /* 620 */ 492, 359, 360, 388, 496, 497, 347, 423, 359, 360, - /* 630 */ 486, 383, 0, 347, 107, 400, 347, 402, 388, 391, - /* 640 */ 77, 379, 467, 468, 469, 395, 471, 472, 379, 174, - /* 650 */ 175, 51, 205, 403, 22, 208, 181, 182, 211, 59, - /* 660 */ 213, 176, 62, 63, 39, 40, 103, 0, 359, 360, - /* 670 */ 20, 196, 22, 198, 439, 471, 174, 442, 474, 400, - /* 680 */ 445, 446, 447, 448, 449, 450, 400, 452, 379, 400, - /* 690 */ 359, 360, 457, 37, 459, 491, 492, 77, 463, 464, - /* 700 */ 496, 497, 228, 228, 229, 55, 231, 232, 233, 234, + /* 480 */ 243, 244, 245, 246, 247, 248, 249, 12, 13, 442, + /* 490 */ 443, 347, 12, 13, 4, 20, 400, 22, 20, 452, + /* 500 */ 347, 359, 360, 195, 360, 197, 72, 73, 74, 42, + /* 510 */ 35, 347, 37, 79, 80, 81, 388, 37, 20, 85, + /* 520 */ 107, 379, 228, 395, 90, 91, 92, 93, 14, 69, + /* 530 */ 96, 403, 388, 99, 20, 227, 46, 47, 48, 64, + /* 540 */ 37, 72, 73, 74, 400, 70, 402, 191, 79, 80, + /* 550 */ 81, 360, 77, 400, 85, 359, 360, 401, 402, 90, + /* 560 */ 91, 92, 93, 347, 400, 96, 359, 360, 99, 174, + /* 570 */ 388, 423, 22, 265, 218, 219, 428, 395, 103, 388, + /* 580 */ 77, 106, 347, 439, 137, 403, 442, 37, 141, 445, + /* 590 */ 446, 447, 448, 449, 450, 360, 452, 362, 249, 471, + /* 600 */ 251, 457, 474, 459, 116, 8, 9, 463, 464, 12, + /* 610 */ 13, 14, 15, 16, 249, 408, 400, 142, 143, 471, + /* 620 */ 492, 493, 474, 388, 22, 497, 498, 77, 359, 360, + /* 630 */ 486, 359, 360, 1, 2, 400, 366, 402, 447, 37, + /* 640 */ 492, 493, 359, 360, 388, 497, 498, 347, 379, 174, + /* 650 */ 175, 379, 205, 103, 176, 208, 181, 182, 211, 403, + /* 660 */ 213, 391, 466, 467, 468, 469, 0, 471, 472, 359, + /* 670 */ 360, 196, 455, 198, 439, 458, 347, 442, 198, 77, + /* 680 */ 445, 446, 447, 448, 449, 450, 388, 452, 174, 379, + /* 690 */ 388, 408, 457, 395, 459, 56, 57, 395, 463, 464, + /* 700 */ 400, 403, 228, 228, 229, 403, 231, 232, 233, 234, /* 710 */ 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, - /* 720 */ 245, 246, 247, 248, 12, 13, 14, 347, 359, 360, - /* 730 */ 388, 388, 20, 77, 22, 110, 111, 395, 113, 408, - /* 740 */ 360, 0, 362, 359, 360, 403, 403, 35, 379, 37, - /* 750 */ 276, 277, 278, 279, 280, 281, 282, 359, 360, 196, - /* 760 */ 135, 198, 116, 379, 139, 8, 9, 444, 388, 12, - /* 770 */ 13, 14, 15, 16, 359, 360, 64, 161, 359, 360, - /* 780 */ 400, 22, 402, 136, 137, 188, 170, 347, 141, 77, - /* 790 */ 77, 228, 229, 470, 379, 368, 37, 347, 379, 14, - /* 800 */ 15, 16, 135, 136, 137, 138, 139, 140, 141, 366, - /* 810 */ 360, 456, 362, 458, 106, 103, 359, 360, 106, 439, - /* 820 */ 347, 394, 442, 401, 402, 445, 446, 447, 448, 449, - /* 830 */ 450, 191, 452, 360, 391, 362, 379, 457, 388, 459, - /* 840 */ 400, 8, 9, 463, 464, 12, 13, 14, 15, 16, - /* 850 */ 400, 456, 402, 458, 142, 143, 56, 57, 218, 219, - /* 860 */ 0, 388, 103, 347, 107, 467, 468, 469, 389, 471, - /* 870 */ 472, 364, 365, 400, 34, 402, 135, 136, 137, 138, - /* 880 */ 139, 140, 141, 64, 359, 360, 174, 175, 291, 439, - /* 890 */ 364, 365, 442, 181, 182, 445, 446, 447, 448, 449, - /* 900 */ 450, 389, 452, 195, 379, 197, 20, 457, 196, 459, - /* 910 */ 198, 0, 439, 463, 464, 442, 400, 389, 445, 446, - /* 920 */ 447, 448, 449, 450, 105, 452, 380, 108, 347, 69, - /* 930 */ 457, 423, 459, 22, 388, 227, 463, 464, 347, 86, - /* 940 */ 228, 229, 396, 231, 232, 233, 234, 235, 236, 237, + /* 720 */ 245, 246, 247, 248, 12, 13, 14, 347, 380, 400, + /* 730 */ 4, 387, 20, 50, 22, 77, 388, 359, 360, 107, + /* 740 */ 360, 0, 362, 399, 396, 19, 196, 35, 198, 37, + /* 750 */ 276, 277, 278, 279, 280, 281, 282, 379, 8, 9, + /* 760 */ 368, 35, 12, 13, 14, 15, 16, 380, 388, 77, + /* 770 */ 64, 14, 15, 16, 34, 388, 64, 51, 228, 229, + /* 780 */ 400, 2, 402, 396, 58, 188, 394, 8, 9, 77, + /* 790 */ 64, 12, 13, 14, 15, 16, 366, 347, 444, 364, + /* 800 */ 365, 135, 136, 137, 138, 139, 140, 141, 364, 365, + /* 810 */ 360, 105, 362, 383, 108, 103, 359, 360, 106, 439, + /* 820 */ 347, 391, 442, 116, 470, 445, 446, 447, 448, 449, + /* 830 */ 450, 105, 452, 360, 108, 362, 379, 457, 388, 459, + /* 840 */ 20, 8, 9, 463, 464, 12, 13, 14, 15, 16, + /* 850 */ 400, 388, 402, 136, 142, 143, 14, 107, 395, 176, + /* 860 */ 0, 388, 20, 347, 8, 9, 403, 184, 12, 13, + /* 870 */ 14, 15, 16, 400, 389, 402, 135, 136, 137, 138, + /* 880 */ 139, 140, 141, 359, 360, 116, 174, 175, 291, 439, + /* 890 */ 0, 22, 442, 181, 182, 445, 446, 447, 448, 449, + /* 900 */ 450, 389, 452, 379, 373, 374, 37, 457, 196, 459, + /* 910 */ 198, 347, 439, 463, 464, 442, 400, 347, 445, 446, + /* 920 */ 447, 448, 449, 450, 347, 452, 209, 210, 423, 3, + /* 930 */ 457, 389, 459, 39, 40, 166, 463, 464, 359, 360, + /* 940 */ 228, 229, 52, 231, 232, 233, 234, 235, 236, 237, /* 950 */ 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - /* 960 */ 248, 12, 13, 376, 389, 347, 347, 373, 374, 20, - /* 970 */ 388, 22, 389, 265, 373, 374, 20, 390, 360, 471, - /* 980 */ 362, 400, 474, 4, 35, 403, 37, 400, 42, 8, - /* 990 */ 9, 400, 347, 12, 13, 14, 15, 16, 19, 491, - /* 1000 */ 492, 148, 359, 360, 496, 497, 388, 173, 455, 176, - /* 1010 */ 33, 458, 397, 64, 35, 400, 359, 360, 400, 400, - /* 1020 */ 402, 2, 379, 170, 359, 360, 77, 8, 9, 442, - /* 1030 */ 51, 12, 13, 14, 15, 16, 379, 58, 380, 452, - /* 1040 */ 347, 359, 360, 64, 379, 400, 388, 423, 397, 389, - /* 1050 */ 347, 400, 103, 389, 396, 106, 347, 439, 33, 376, - /* 1060 */ 442, 379, 176, 445, 446, 447, 448, 449, 450, 360, - /* 1070 */ 452, 362, 70, 390, 347, 457, 347, 459, 350, 351, - /* 1080 */ 348, 463, 464, 400, 105, 0, 252, 108, 107, 347, - /* 1090 */ 347, 142, 143, 400, 2, 471, 262, 388, 474, 347, - /* 1100 */ 8, 9, 423, 400, 12, 13, 14, 15, 16, 400, - /* 1110 */ 212, 402, 214, 388, 397, 491, 492, 400, 429, 381, - /* 1120 */ 496, 497, 384, 174, 175, 442, 443, 400, 403, 400, - /* 1130 */ 181, 182, 176, 8, 9, 452, 51, 12, 13, 14, - /* 1140 */ 15, 16, 400, 400, 33, 196, 375, 198, 439, 378, - /* 1150 */ 471, 442, 400, 474, 445, 446, 447, 448, 449, 450, - /* 1160 */ 33, 452, 47, 48, 263, 264, 457, 33, 459, 0, - /* 1170 */ 491, 492, 463, 464, 116, 496, 497, 228, 229, 45, + /* 960 */ 248, 12, 13, 107, 400, 347, 423, 4, 423, 20, + /* 970 */ 400, 22, 103, 444, 380, 347, 471, 400, 360, 474, + /* 980 */ 362, 2, 388, 456, 35, 458, 37, 8, 9, 42, + /* 990 */ 396, 12, 13, 14, 15, 16, 176, 492, 493, 470, + /* 1000 */ 347, 33, 497, 498, 110, 111, 388, 113, 20, 176, + /* 1010 */ 423, 359, 360, 64, 471, 33, 471, 474, 400, 474, + /* 1020 */ 402, 161, 373, 374, 359, 360, 77, 45, 400, 135, + /* 1030 */ 170, 379, 456, 139, 458, 492, 493, 492, 493, 22, + /* 1040 */ 497, 498, 497, 498, 136, 137, 467, 468, 469, 141, + /* 1050 */ 471, 472, 103, 400, 37, 106, 347, 439, 471, 397, + /* 1060 */ 442, 474, 400, 445, 446, 447, 448, 449, 450, 360, + /* 1070 */ 452, 362, 347, 359, 360, 457, 347, 459, 20, 492, + /* 1080 */ 493, 463, 464, 389, 497, 498, 359, 360, 388, 388, + /* 1090 */ 429, 142, 143, 379, 8, 9, 347, 388, 12, 13, + /* 1100 */ 14, 15, 16, 403, 403, 33, 379, 359, 360, 400, + /* 1110 */ 347, 402, 347, 0, 347, 359, 360, 45, 359, 360, + /* 1120 */ 103, 359, 360, 174, 175, 400, 0, 379, 13, 400, + /* 1130 */ 181, 182, 467, 468, 469, 379, 471, 472, 379, 350, + /* 1140 */ 351, 379, 359, 360, 33, 196, 33, 198, 439, 400, + /* 1150 */ 397, 442, 37, 400, 445, 446, 447, 448, 449, 450, + /* 1160 */ 33, 452, 379, 400, 176, 400, 457, 400, 459, 47, + /* 1170 */ 48, 397, 463, 464, 400, 33, 381, 228, 229, 384, /* 1180 */ 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, /* 1190 */ 241, 242, 243, 244, 245, 246, 247, 248, 12, 13, - /* 1200 */ 33, 13, 347, 13, 109, 33, 20, 112, 22, 0, - /* 1210 */ 264, 42, 45, 109, 109, 360, 112, 112, 107, 109, - /* 1220 */ 64, 35, 112, 37, 166, 37, 49, 37, 142, 143, - /* 1230 */ 228, 22, 33, 37, 107, 347, 33, 33, 37, 33, - /* 1240 */ 377, 1, 2, 388, 106, 37, 33, 33, 360, 33, - /* 1250 */ 64, 33, 33, 115, 413, 400, 500, 402, 33, 489, - /* 1260 */ 363, 483, 13, 77, 108, 288, 376, 13, 376, 33, - /* 1270 */ 33, 33, 0, 77, 422, 33, 388, 33, 363, 107, - /* 1280 */ 33, 33, 388, 106, 413, 358, 37, 413, 400, 103, - /* 1290 */ 402, 37, 106, 360, 439, 399, 473, 442, 493, 268, - /* 1300 */ 445, 446, 447, 448, 449, 450, 107, 452, 465, 424, - /* 1310 */ 107, 107, 457, 107, 459, 290, 51, 441, 463, 464, - /* 1320 */ 107, 107, 476, 107, 52, 107, 107, 439, 142, 143, - /* 1330 */ 442, 42, 107, 445, 446, 447, 448, 449, 450, 451, - /* 1340 */ 452, 453, 454, 107, 107, 107, 440, 18, 20, 107, - /* 1350 */ 433, 107, 23, 211, 107, 107, 438, 368, 433, 368, - /* 1360 */ 174, 175, 194, 426, 20, 359, 20, 181, 182, 40, - /* 1370 */ 41, 360, 45, 44, 409, 20, 413, 360, 409, 406, - /* 1380 */ 173, 359, 196, 54, 198, 409, 360, 359, 406, 406, - /* 1390 */ 104, 359, 372, 102, 65, 66, 67, 68, 371, 198, - /* 1400 */ 101, 359, 359, 370, 359, 20, 198, 352, 50, 356, - /* 1410 */ 356, 352, 368, 433, 228, 229, 20, 231, 232, 233, + /* 1200 */ 375, 13, 347, 378, 263, 264, 20, 109, 22, 0, + /* 1210 */ 112, 264, 286, 109, 109, 360, 112, 112, 107, 109, + /* 1220 */ 0, 35, 112, 37, 212, 37, 214, 64, 33, 49, + /* 1230 */ 267, 22, 174, 37, 107, 347, 33, 33, 37, 33, + /* 1240 */ 13, 33, 22, 388, 142, 143, 33, 33, 360, 107, + /* 1250 */ 64, 33, 33, 1, 2, 400, 288, 402, 33, 389, + /* 1260 */ 377, 348, 413, 77, 37, 501, 490, 13, 363, 33, + /* 1270 */ 33, 108, 376, 77, 483, 33, 388, 33, 376, 33, + /* 1280 */ 33, 33, 33, 489, 422, 489, 106, 37, 400, 103, + /* 1290 */ 402, 37, 106, 489, 439, 388, 489, 442, 363, 413, + /* 1300 */ 445, 446, 447, 448, 449, 450, 358, 452, 399, 473, + /* 1310 */ 107, 107, 457, 107, 459, 107, 360, 413, 463, 464, + /* 1320 */ 107, 107, 494, 465, 476, 107, 107, 439, 142, 143, + /* 1330 */ 442, 268, 107, 445, 446, 447, 448, 449, 450, 451, + /* 1340 */ 452, 453, 454, 107, 107, 424, 51, 18, 222, 107, + /* 1350 */ 42, 107, 23, 107, 107, 107, 107, 441, 440, 20, + /* 1360 */ 174, 175, 433, 211, 368, 438, 433, 181, 182, 40, + /* 1370 */ 41, 368, 194, 44, 20, 359, 426, 20, 360, 45, + /* 1380 */ 409, 20, 196, 54, 198, 413, 360, 409, 173, 406, + /* 1390 */ 359, 406, 360, 359, 65, 66, 67, 68, 409, 198, + /* 1400 */ 406, 104, 372, 102, 371, 359, 101, 370, 359, 359, + /* 1410 */ 359, 20, 352, 50, 228, 229, 352, 231, 232, 233, /* 1420 */ 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, - /* 1430 */ 244, 245, 246, 247, 248, 106, 12, 13, 402, 368, - /* 1440 */ 20, 361, 20, 425, 368, 361, 22, 368, 20, 368, - /* 1450 */ 416, 368, 359, 352, 388, 368, 388, 388, 388, 35, - /* 1460 */ 388, 37, 388, 388, 350, 388, 388, 359, 350, 352, - /* 1470 */ 388, 347, 215, 144, 437, 106, 388, 202, 366, 201, - /* 1480 */ 433, 435, 432, 431, 360, 366, 430, 200, 64, 400, - /* 1490 */ 400, 400, 359, 283, 482, 275, 402, 424, 400, 400, - /* 1500 */ 274, 77, 400, 482, 187, 481, 400, 347, 485, 413, - /* 1510 */ 413, 284, 388, 484, 285, 480, 292, 188, 189, 190, - /* 1520 */ 360, 418, 193, 482, 400, 418, 402, 103, 269, 20, - /* 1530 */ 479, 501, 289, 424, 495, 206, 207, 287, 347, 360, - /* 1540 */ 264, 116, 494, 266, 444, 361, 217, 366, 388, 220, - /* 1550 */ 418, 360, 223, 224, 225, 226, 227, 400, 366, 400, - /* 1560 */ 400, 400, 402, 439, 400, 400, 442, 418, 179, 445, - /* 1570 */ 446, 447, 448, 449, 450, 400, 452, 414, 366, 388, - /* 1580 */ 106, 457, 384, 459, 462, 106, 392, 463, 464, 366, - /* 1590 */ 477, 400, 360, 402, 265, 475, 366, 400, 359, 439, - /* 1600 */ 378, 22, 442, 347, 349, 445, 446, 447, 448, 449, - /* 1610 */ 450, 38, 452, 353, 352, 382, 360, 457, 367, 459, - /* 1620 */ 196, 434, 198, 463, 464, 345, 0, 427, 382, 0, - /* 1630 */ 439, 0, 45, 442, 419, 382, 445, 446, 447, 448, - /* 1640 */ 449, 450, 0, 452, 388, 37, 37, 419, 221, 37, - /* 1650 */ 459, 37, 228, 229, 463, 464, 400, 0, 402, 221, - /* 1660 */ 37, 37, 221, 0, 37, 241, 242, 243, 244, 245, - /* 1670 */ 246, 247, 221, 0, 37, 0, 22, 347, 0, 216, - /* 1680 */ 37, 0, 204, 0, 204, 198, 205, 196, 0, 0, - /* 1690 */ 360, 0, 192, 0, 191, 439, 0, 147, 442, 49, - /* 1700 */ 347, 445, 446, 447, 448, 449, 450, 49, 452, 0, - /* 1710 */ 51, 37, 0, 360, 49, 459, 45, 0, 388, 463, - /* 1720 */ 464, 0, 0, 0, 0, 49, 0, 0, 0, 0, - /* 1730 */ 400, 161, 402, 0, 37, 0, 161, 0, 0, 0, + /* 1430 */ 244, 245, 246, 247, 248, 106, 12, 13, 356, 356, + /* 1440 */ 433, 368, 20, 402, 368, 20, 22, 20, 198, 361, + /* 1450 */ 425, 361, 368, 368, 20, 416, 359, 368, 352, 35, + /* 1460 */ 368, 37, 368, 388, 350, 388, 388, 388, 388, 388, + /* 1470 */ 350, 347, 215, 144, 400, 388, 359, 352, 106, 388, + /* 1480 */ 437, 388, 435, 388, 360, 290, 433, 388, 64, 366, + /* 1490 */ 202, 432, 201, 431, 366, 430, 400, 359, 200, 275, + /* 1500 */ 400, 77, 482, 402, 274, 482, 283, 347, 187, 400, + /* 1510 */ 481, 484, 388, 485, 285, 284, 413, 188, 189, 190, + /* 1520 */ 360, 413, 193, 479, 400, 480, 402, 103, 269, 264, + /* 1530 */ 418, 424, 292, 360, 418, 206, 207, 482, 347, 20, + /* 1540 */ 502, 266, 424, 495, 289, 287, 217, 116, 388, 220, + /* 1550 */ 496, 360, 223, 224, 225, 226, 227, 418, 361, 366, + /* 1560 */ 400, 400, 402, 439, 366, 444, 442, 400, 400, 445, + /* 1570 */ 446, 447, 448, 449, 450, 400, 452, 418, 179, 388, + /* 1580 */ 414, 457, 400, 459, 384, 106, 106, 463, 464, 400, + /* 1590 */ 378, 400, 366, 402, 265, 360, 477, 462, 366, 439, + /* 1600 */ 359, 22, 442, 347, 475, 445, 446, 447, 448, 449, + /* 1610 */ 450, 366, 452, 392, 38, 349, 360, 457, 434, 459, + /* 1620 */ 196, 353, 198, 463, 464, 352, 427, 345, 367, 419, + /* 1630 */ 439, 0, 0, 442, 0, 419, 445, 446, 447, 448, + /* 1640 */ 449, 450, 382, 452, 388, 382, 382, 45, 0, 37, + /* 1650 */ 459, 221, 228, 229, 463, 464, 400, 37, 402, 37, + /* 1660 */ 37, 221, 0, 37, 37, 241, 242, 243, 244, 245, + /* 1670 */ 246, 247, 221, 37, 0, 221, 0, 347, 37, 0, + /* 1680 */ 22, 0, 37, 216, 0, 204, 0, 204, 198, 205, + /* 1690 */ 360, 196, 0, 0, 0, 439, 192, 191, 442, 0, + /* 1700 */ 347, 445, 446, 447, 448, 449, 450, 0, 452, 147, + /* 1710 */ 49, 0, 49, 360, 37, 459, 51, 0, 388, 463, + /* 1720 */ 464, 49, 0, 45, 0, 0, 0, 0, 0, 49, + /* 1730 */ 400, 161, 402, 0, 0, 0, 0, 37, 0, 161, /* 1740 */ 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, - /* 1750 */ 0, 0, 0, 400, 0, 402, 49, 0, 45, 0, - /* 1760 */ 0, 0, 0, 0, 0, 147, 0, 146, 347, 439, - /* 1770 */ 0, 0, 442, 145, 0, 445, 446, 447, 448, 449, - /* 1780 */ 450, 360, 452, 347, 0, 0, 50, 22, 0, 459, - /* 1790 */ 22, 22, 439, 463, 464, 442, 360, 0, 445, 446, - /* 1800 */ 447, 448, 449, 450, 37, 452, 50, 0, 0, 388, - /* 1810 */ 37, 0, 0, 37, 42, 0, 37, 64, 37, 64, - /* 1820 */ 0, 400, 64, 402, 388, 42, 51, 45, 35, 51, - /* 1830 */ 14, 51, 42, 42, 33, 49, 400, 0, 402, 0, - /* 1840 */ 487, 488, 43, 0, 51, 49, 42, 49, 0, 347, - /* 1850 */ 0, 42, 59, 60, 61, 62, 187, 64, 0, 49, + /* 1750 */ 0, 0, 0, 400, 0, 402, 0, 0, 0, 49, + /* 1760 */ 0, 0, 0, 45, 0, 0, 0, 0, 347, 439, + /* 1770 */ 0, 147, 442, 0, 146, 445, 446, 447, 448, 449, + /* 1780 */ 450, 360, 452, 347, 0, 22, 0, 145, 0, 459, + /* 1790 */ 22, 50, 439, 463, 464, 442, 360, 22, 445, 446, + /* 1800 */ 447, 448, 449, 450, 0, 452, 0, 0, 37, 388, + /* 1810 */ 0, 0, 0, 37, 42, 0, 0, 64, 37, 50, + /* 1820 */ 37, 400, 0, 402, 388, 51, 37, 42, 35, 42, + /* 1830 */ 33, 51, 14, 51, 42, 45, 400, 0, 402, 0, + /* 1840 */ 487, 488, 64, 64, 51, 49, 43, 49, 49, 347, + /* 1850 */ 42, 0, 59, 60, 61, 62, 0, 64, 187, 42, /* 1860 */ 439, 0, 360, 442, 0, 0, 445, 446, 447, 448, - /* 1870 */ 449, 450, 37, 452, 71, 439, 51, 0, 442, 37, - /* 1880 */ 42, 445, 446, 447, 448, 449, 450, 51, 452, 42, - /* 1890 */ 388, 0, 37, 42, 0, 459, 51, 37, 105, 42, - /* 1900 */ 464, 108, 400, 51, 402, 0, 0, 0, 0, 0, - /* 1910 */ 0, 37, 112, 0, 347, 22, 114, 37, 37, 498, - /* 1920 */ 499, 37, 37, 37, 37, 37, 37, 360, 33, 347, - /* 1930 */ 33, 0, 0, 37, 22, 37, 37, 0, 22, 22, - /* 1940 */ 22, 439, 360, 53, 442, 0, 0, 445, 446, 447, - /* 1950 */ 448, 449, 450, 0, 452, 388, 37, 37, 0, 37, - /* 1960 */ 393, 0, 22, 20, 37, 37, 37, 400, 106, 402, - /* 1970 */ 388, 178, 179, 107, 0, 393, 106, 22, 185, 186, - /* 1980 */ 203, 176, 400, 0, 402, 347, 37, 22, 199, 176, - /* 1990 */ 488, 0, 0, 179, 176, 176, 3, 204, 360, 33, - /* 2000 */ 183, 50, 270, 49, 183, 33, 439, 106, 106, 442, - /* 2010 */ 107, 107, 445, 446, 447, 448, 449, 450, 104, 452, - /* 2020 */ 50, 439, 33, 102, 442, 49, 388, 445, 446, 447, - /* 2030 */ 448, 449, 450, 107, 452, 33, 106, 33, 400, 106, - /* 2040 */ 402, 106, 3, 107, 106, 33, 107, 37, 347, 107, - /* 2050 */ 37, 37, 37, 37, 37, 49, 33, 270, 263, 107, - /* 2060 */ 49, 360, 270, 347, 107, 0, 0, 42, 106, 106, - /* 2070 */ 0, 107, 180, 42, 107, 106, 360, 439, 347, 106, - /* 2080 */ 442, 106, 49, 445, 446, 447, 448, 449, 450, 388, - /* 2090 */ 452, 360, 115, 106, 33, 2, 250, 22, 104, 104, - /* 2100 */ 228, 400, 178, 402, 388, 106, 49, 107, 107, 393, - /* 2110 */ 107, 106, 49, 106, 106, 106, 400, 22, 402, 388, - /* 2120 */ 37, 37, 37, 37, 37, 107, 106, 106, 490, 107, - /* 2130 */ 230, 400, 106, 402, 107, 116, 107, 107, 106, 37, - /* 2140 */ 439, 347, 106, 442, 107, 107, 445, 446, 447, 448, - /* 2150 */ 449, 450, 106, 452, 360, 439, 37, 106, 442, 106, - /* 2160 */ 33, 445, 446, 447, 448, 449, 450, 127, 452, 127, - /* 2170 */ 439, 347, 127, 442, 127, 106, 445, 446, 447, 448, - /* 2180 */ 449, 450, 388, 452, 360, 454, 37, 393, 106, 22, - /* 2190 */ 37, 71, 70, 37, 400, 37, 402, 37, 1, 37, - /* 2200 */ 499, 37, 37, 37, 100, 37, 77, 33, 37, 37, - /* 2210 */ 37, 22, 388, 37, 37, 37, 19, 393, 77, 37, - /* 2220 */ 37, 37, 22, 37, 400, 37, 402, 37, 0, 37, - /* 2230 */ 51, 0, 35, 439, 0, 42, 442, 42, 37, 445, - /* 2240 */ 446, 447, 448, 449, 450, 51, 452, 37, 51, 51, - /* 2250 */ 42, 0, 37, 51, 42, 0, 59, 60, 61, 62, - /* 2260 */ 37, 64, 347, 439, 37, 0, 442, 22, 33, 445, - /* 2270 */ 446, 447, 448, 449, 450, 360, 452, 20, 22, 21, - /* 2280 */ 502, 22, 22, 21, 502, 502, 502, 502, 502, 502, - /* 2290 */ 502, 502, 502, 502, 502, 347, 502, 502, 502, 502, - /* 2300 */ 502, 502, 105, 388, 502, 108, 502, 502, 360, 502, - /* 2310 */ 502, 502, 502, 502, 502, 400, 502, 402, 347, 502, - /* 2320 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2330 */ 502, 360, 502, 347, 502, 502, 388, 140, 502, 502, - /* 2340 */ 502, 502, 502, 502, 502, 502, 360, 502, 400, 502, - /* 2350 */ 402, 502, 502, 502, 439, 502, 502, 442, 502, 388, - /* 2360 */ 445, 446, 447, 448, 449, 450, 502, 452, 502, 502, - /* 2370 */ 502, 400, 502, 402, 388, 178, 502, 502, 502, 502, - /* 2380 */ 502, 502, 185, 502, 502, 502, 400, 439, 402, 502, - /* 2390 */ 442, 502, 502, 445, 446, 447, 448, 449, 450, 502, - /* 2400 */ 452, 204, 502, 502, 502, 502, 502, 502, 502, 347, - /* 2410 */ 439, 502, 502, 442, 502, 502, 445, 446, 447, 448, - /* 2420 */ 449, 450, 360, 452, 347, 439, 502, 502, 442, 502, - /* 2430 */ 502, 445, 446, 447, 448, 449, 450, 360, 452, 347, - /* 2440 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2450 */ 388, 502, 360, 502, 502, 502, 502, 502, 502, 502, - /* 2460 */ 502, 502, 400, 502, 402, 388, 502, 502, 502, 502, - /* 2470 */ 502, 502, 502, 502, 502, 502, 502, 400, 502, 402, - /* 2480 */ 388, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2490 */ 502, 502, 400, 502, 402, 502, 502, 502, 502, 502, - /* 2500 */ 502, 439, 347, 502, 442, 502, 502, 445, 446, 447, - /* 2510 */ 448, 449, 450, 502, 452, 360, 439, 502, 502, 442, - /* 2520 */ 502, 502, 445, 446, 447, 448, 449, 450, 502, 452, - /* 2530 */ 502, 439, 347, 502, 442, 502, 502, 445, 446, 447, - /* 2540 */ 448, 449, 450, 388, 452, 360, 502, 502, 502, 502, - /* 2550 */ 502, 502, 502, 502, 502, 400, 502, 402, 502, 502, - /* 2560 */ 502, 502, 502, 502, 502, 347, 502, 502, 502, 502, - /* 2570 */ 502, 502, 502, 388, 502, 502, 502, 502, 360, 502, - /* 2580 */ 502, 502, 502, 502, 502, 400, 502, 402, 502, 502, - /* 2590 */ 502, 502, 502, 502, 439, 502, 502, 442, 502, 502, - /* 2600 */ 445, 446, 447, 448, 449, 450, 388, 452, 502, 502, - /* 2610 */ 502, 502, 502, 502, 502, 502, 502, 502, 400, 502, - /* 2620 */ 402, 502, 502, 502, 439, 502, 502, 442, 502, 502, - /* 2630 */ 445, 446, 447, 448, 449, 450, 502, 452, 502, 502, - /* 2640 */ 502, 502, 502, 347, 502, 502, 502, 502, 502, 502, - /* 2650 */ 502, 502, 502, 502, 502, 502, 360, 439, 502, 502, - /* 2660 */ 442, 502, 502, 445, 446, 447, 448, 449, 450, 502, - /* 2670 */ 452, 502, 502, 502, 502, 502, 502, 502, 502, 347, - /* 2680 */ 502, 502, 502, 502, 388, 502, 502, 502, 502, 502, - /* 2690 */ 502, 502, 360, 502, 502, 502, 400, 502, 402, 502, - /* 2700 */ 502, 502, 502, 502, 502, 502, 502, 502, 347, 502, - /* 2710 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2720 */ 388, 360, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2730 */ 502, 502, 400, 502, 402, 439, 502, 502, 442, 502, - /* 2740 */ 502, 445, 446, 447, 448, 449, 450, 502, 452, 388, - /* 2750 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2760 */ 502, 400, 502, 402, 502, 502, 502, 502, 502, 502, - /* 2770 */ 502, 439, 502, 347, 442, 502, 502, 445, 446, 447, - /* 2780 */ 448, 449, 450, 502, 452, 502, 360, 502, 502, 502, - /* 2790 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2800 */ 439, 502, 502, 442, 502, 502, 445, 446, 447, 448, - /* 2810 */ 449, 450, 502, 452, 388, 502, 502, 502, 502, 502, - /* 2820 */ 502, 502, 502, 502, 502, 502, 400, 502, 402, 502, - /* 2830 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2840 */ 347, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2850 */ 502, 502, 502, 360, 502, 502, 347, 502, 502, 502, - /* 2860 */ 502, 502, 502, 502, 502, 439, 502, 502, 442, 360, - /* 2870 */ 502, 445, 446, 447, 448, 449, 450, 502, 452, 502, - /* 2880 */ 502, 388, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2890 */ 502, 502, 502, 400, 502, 402, 502, 388, 502, 502, - /* 2900 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 400, - /* 2910 */ 502, 402, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2920 */ 502, 347, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2930 */ 502, 502, 439, 502, 360, 442, 502, 502, 445, 446, - /* 2940 */ 447, 448, 449, 450, 502, 452, 502, 502, 439, 502, - /* 2950 */ 347, 442, 502, 502, 445, 446, 447, 448, 449, 450, - /* 2960 */ 502, 452, 388, 360, 502, 502, 502, 502, 502, 502, - /* 2970 */ 502, 502, 502, 502, 400, 502, 402, 502, 502, 502, - /* 2980 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 2990 */ 502, 388, 502, 502, 502, 502, 502, 502, 502, 502, - /* 3000 */ 502, 502, 502, 400, 502, 402, 502, 502, 502, 502, - /* 3010 */ 502, 502, 502, 439, 502, 347, 442, 502, 502, 445, - /* 3020 */ 446, 447, 448, 449, 450, 502, 452, 502, 360, 502, - /* 3030 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 3040 */ 502, 502, 439, 502, 347, 442, 502, 502, 445, 446, - /* 3050 */ 447, 448, 449, 450, 502, 452, 388, 360, 502, 502, - /* 3060 */ 502, 502, 502, 502, 502, 502, 502, 502, 400, 502, - /* 3070 */ 402, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 3080 */ 347, 502, 502, 502, 502, 388, 502, 502, 502, 502, - /* 3090 */ 502, 502, 502, 360, 502, 502, 502, 400, 502, 402, - /* 3100 */ 502, 502, 502, 502, 502, 502, 502, 439, 502, 347, - /* 3110 */ 442, 502, 502, 445, 446, 447, 448, 449, 450, 502, - /* 3120 */ 452, 388, 360, 502, 502, 502, 502, 502, 502, 502, - /* 3130 */ 502, 502, 502, 400, 502, 402, 439, 502, 502, 442, - /* 3140 */ 502, 502, 445, 446, 447, 448, 449, 450, 502, 452, - /* 3150 */ 388, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 3160 */ 502, 502, 400, 502, 402, 502, 502, 502, 502, 502, - /* 3170 */ 502, 502, 439, 502, 502, 442, 502, 502, 445, 446, - /* 3180 */ 447, 448, 449, 450, 502, 452, 502, 502, 502, 502, - /* 3190 */ 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - /* 3200 */ 502, 439, 502, 502, 442, 502, 502, 445, 446, 447, - /* 3210 */ 448, 449, 450, 502, 452, 344, 344, 344, 344, 344, + /* 1870 */ 449, 450, 49, 452, 0, 439, 71, 0, 442, 37, + /* 1880 */ 51, 445, 446, 447, 448, 449, 450, 42, 452, 0, + /* 1890 */ 388, 37, 51, 0, 42, 459, 37, 51, 105, 42, + /* 1900 */ 464, 108, 400, 0, 402, 37, 51, 0, 42, 0, + /* 1910 */ 0, 0, 0, 0, 347, 114, 37, 22, 0, 37, + /* 1920 */ 499, 500, 37, 37, 37, 112, 22, 360, 37, 347, + /* 1930 */ 0, 37, 33, 37, 37, 33, 22, 37, 37, 0, + /* 1940 */ 0, 439, 360, 37, 442, 22, 22, 445, 446, 447, + /* 1950 */ 448, 449, 450, 37, 452, 388, 0, 0, 53, 0, + /* 1960 */ 393, 0, 37, 37, 0, 22, 20, 400, 37, 402, + /* 1970 */ 388, 178, 179, 37, 37, 393, 107, 106, 185, 186, + /* 1980 */ 0, 106, 400, 0, 402, 347, 37, 22, 22, 0, + /* 1990 */ 488, 0, 183, 179, 203, 176, 176, 204, 360, 199, + /* 2000 */ 3, 176, 176, 33, 183, 107, 439, 270, 50, 442, + /* 2010 */ 50, 33, 445, 446, 447, 448, 449, 450, 33, 452, + /* 2020 */ 33, 439, 102, 106, 442, 106, 388, 445, 446, 447, + /* 2030 */ 448, 449, 450, 104, 452, 107, 33, 49, 400, 107, + /* 2040 */ 402, 106, 49, 3, 106, 33, 37, 106, 347, 107, + /* 2050 */ 106, 37, 107, 37, 107, 37, 37, 37, 107, 107, + /* 2060 */ 49, 360, 33, 347, 49, 270, 0, 0, 106, 42, + /* 2070 */ 107, 0, 270, 106, 42, 107, 360, 439, 347, 106, + /* 2080 */ 442, 106, 106, 445, 446, 447, 448, 449, 450, 388, + /* 2090 */ 452, 360, 115, 263, 49, 2, 106, 33, 104, 178, + /* 2100 */ 250, 400, 180, 402, 388, 104, 22, 107, 228, 393, + /* 2110 */ 106, 106, 49, 107, 106, 116, 400, 107, 402, 388, + /* 2120 */ 107, 49, 22, 106, 37, 37, 37, 106, 106, 491, + /* 2130 */ 107, 400, 106, 402, 37, 107, 106, 106, 230, 37, + /* 2140 */ 439, 347, 107, 442, 107, 106, 445, 446, 447, 448, + /* 2150 */ 449, 450, 37, 452, 360, 439, 107, 106, 442, 107, + /* 2160 */ 37, 445, 446, 447, 448, 449, 450, 106, 452, 127, + /* 2170 */ 439, 347, 33, 442, 127, 127, 445, 446, 447, 448, + /* 2180 */ 449, 450, 388, 452, 360, 454, 127, 393, 106, 106, + /* 2190 */ 37, 106, 22, 71, 400, 70, 402, 37, 1, 37, + /* 2200 */ 37, 500, 37, 37, 37, 37, 37, 77, 37, 100, + /* 2210 */ 33, 37, 388, 37, 22, 37, 19, 393, 37, 37, + /* 2220 */ 37, 77, 22, 37, 400, 37, 402, 37, 37, 0, + /* 2230 */ 37, 37, 35, 439, 0, 37, 442, 42, 51, 445, + /* 2240 */ 446, 447, 448, 449, 450, 37, 452, 51, 51, 42, + /* 2250 */ 0, 37, 51, 42, 0, 51, 59, 60, 61, 62, + /* 2260 */ 37, 64, 347, 439, 42, 0, 442, 37, 37, 445, + /* 2270 */ 446, 447, 448, 449, 450, 360, 452, 0, 22, 33, + /* 2280 */ 22, 21, 503, 22, 22, 21, 20, 503, 503, 503, + /* 2290 */ 503, 503, 503, 503, 503, 347, 503, 503, 503, 503, + /* 2300 */ 503, 503, 105, 388, 503, 108, 503, 503, 360, 503, + /* 2310 */ 503, 503, 503, 503, 503, 400, 503, 402, 347, 503, + /* 2320 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2330 */ 503, 360, 503, 347, 503, 503, 388, 140, 503, 503, + /* 2340 */ 503, 503, 503, 503, 503, 503, 360, 503, 400, 503, + /* 2350 */ 402, 503, 503, 503, 439, 503, 503, 442, 503, 388, + /* 2360 */ 445, 446, 447, 448, 449, 450, 503, 452, 503, 503, + /* 2370 */ 503, 400, 503, 402, 388, 178, 503, 503, 503, 503, + /* 2380 */ 503, 503, 185, 503, 503, 503, 400, 439, 402, 503, + /* 2390 */ 442, 503, 503, 445, 446, 447, 448, 449, 450, 503, + /* 2400 */ 452, 204, 503, 503, 503, 503, 503, 503, 503, 347, + /* 2410 */ 439, 503, 503, 442, 503, 503, 445, 446, 447, 448, + /* 2420 */ 449, 450, 360, 452, 347, 439, 503, 503, 442, 503, + /* 2430 */ 503, 445, 446, 447, 448, 449, 450, 360, 452, 347, + /* 2440 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2450 */ 388, 503, 360, 503, 503, 503, 503, 503, 503, 503, + /* 2460 */ 503, 503, 400, 503, 402, 388, 503, 503, 503, 503, + /* 2470 */ 503, 503, 503, 503, 503, 503, 503, 400, 503, 402, + /* 2480 */ 388, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2490 */ 503, 503, 400, 503, 402, 503, 503, 503, 503, 503, + /* 2500 */ 503, 439, 347, 503, 442, 503, 503, 445, 446, 447, + /* 2510 */ 448, 449, 450, 503, 452, 360, 439, 503, 503, 442, + /* 2520 */ 503, 503, 445, 446, 447, 448, 449, 450, 503, 452, + /* 2530 */ 503, 439, 347, 503, 442, 503, 503, 445, 446, 447, + /* 2540 */ 448, 449, 450, 388, 452, 360, 503, 503, 503, 503, + /* 2550 */ 503, 503, 503, 503, 503, 400, 503, 402, 503, 503, + /* 2560 */ 503, 503, 503, 503, 503, 347, 503, 503, 503, 503, + /* 2570 */ 503, 503, 503, 388, 503, 503, 503, 503, 360, 503, + /* 2580 */ 503, 503, 503, 503, 503, 400, 503, 402, 503, 503, + /* 2590 */ 503, 503, 503, 503, 439, 503, 503, 442, 503, 503, + /* 2600 */ 445, 446, 447, 448, 449, 450, 388, 452, 503, 503, + /* 2610 */ 503, 503, 503, 503, 503, 503, 503, 503, 400, 503, + /* 2620 */ 402, 503, 503, 503, 439, 503, 503, 442, 503, 503, + /* 2630 */ 445, 446, 447, 448, 449, 450, 503, 452, 503, 503, + /* 2640 */ 503, 503, 503, 347, 503, 503, 503, 503, 503, 503, + /* 2650 */ 503, 503, 503, 503, 503, 503, 360, 439, 503, 503, + /* 2660 */ 442, 503, 503, 445, 446, 447, 448, 449, 450, 503, + /* 2670 */ 452, 503, 503, 503, 503, 503, 503, 503, 503, 347, + /* 2680 */ 503, 503, 503, 503, 388, 503, 503, 503, 503, 503, + /* 2690 */ 503, 503, 360, 503, 503, 503, 400, 503, 402, 503, + /* 2700 */ 503, 503, 503, 503, 503, 503, 503, 503, 347, 503, + /* 2710 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2720 */ 388, 360, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2730 */ 503, 503, 400, 503, 402, 439, 503, 503, 442, 503, + /* 2740 */ 503, 445, 446, 447, 448, 449, 450, 503, 452, 388, + /* 2750 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2760 */ 503, 400, 503, 402, 503, 503, 503, 503, 503, 503, + /* 2770 */ 503, 439, 503, 347, 442, 503, 503, 445, 446, 447, + /* 2780 */ 448, 449, 450, 503, 452, 503, 360, 503, 503, 503, + /* 2790 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2800 */ 439, 503, 503, 442, 503, 503, 445, 446, 447, 448, + /* 2810 */ 449, 450, 503, 452, 388, 503, 503, 503, 503, 503, + /* 2820 */ 503, 503, 503, 503, 503, 503, 400, 503, 402, 503, + /* 2830 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2840 */ 347, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2850 */ 503, 503, 503, 360, 503, 503, 347, 503, 503, 503, + /* 2860 */ 503, 503, 503, 503, 503, 439, 503, 503, 442, 360, + /* 2870 */ 503, 445, 446, 447, 448, 449, 450, 503, 452, 503, + /* 2880 */ 503, 388, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2890 */ 503, 503, 503, 400, 503, 402, 503, 388, 503, 503, + /* 2900 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 400, + /* 2910 */ 503, 402, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2920 */ 503, 347, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2930 */ 503, 503, 439, 503, 360, 442, 503, 503, 445, 446, + /* 2940 */ 447, 448, 449, 450, 503, 452, 503, 503, 439, 503, + /* 2950 */ 347, 442, 503, 503, 445, 446, 447, 448, 449, 450, + /* 2960 */ 503, 452, 388, 360, 503, 503, 503, 503, 503, 503, + /* 2970 */ 503, 503, 503, 503, 400, 503, 402, 503, 503, 503, + /* 2980 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 2990 */ 503, 388, 503, 503, 503, 503, 503, 503, 503, 503, + /* 3000 */ 503, 503, 503, 400, 503, 402, 503, 503, 503, 503, + /* 3010 */ 503, 503, 503, 439, 503, 347, 442, 503, 503, 445, + /* 3020 */ 446, 447, 448, 449, 450, 503, 452, 503, 360, 503, + /* 3030 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 3040 */ 503, 503, 439, 503, 347, 442, 503, 503, 445, 446, + /* 3050 */ 447, 448, 449, 450, 503, 452, 388, 360, 503, 503, + /* 3060 */ 503, 503, 503, 503, 503, 503, 503, 503, 400, 503, + /* 3070 */ 402, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 3080 */ 347, 503, 503, 503, 503, 388, 503, 503, 503, 503, + /* 3090 */ 503, 503, 503, 360, 503, 503, 503, 400, 503, 402, + /* 3100 */ 503, 503, 503, 503, 503, 503, 503, 439, 503, 347, + /* 3110 */ 442, 503, 503, 445, 446, 447, 448, 449, 450, 503, + /* 3120 */ 452, 388, 360, 503, 503, 503, 503, 503, 503, 503, + /* 3130 */ 503, 503, 503, 400, 503, 402, 439, 503, 503, 442, + /* 3140 */ 503, 503, 445, 446, 447, 448, 449, 450, 503, 452, + /* 3150 */ 388, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 3160 */ 503, 503, 400, 503, 402, 503, 503, 503, 503, 503, + /* 3170 */ 503, 503, 439, 503, 503, 442, 503, 503, 445, 446, + /* 3180 */ 447, 448, 449, 450, 503, 452, 503, 503, 503, 503, + /* 3190 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + /* 3200 */ 503, 439, 503, 503, 442, 503, 503, 445, 446, 447, + /* 3210 */ 448, 449, 450, 503, 452, 344, 344, 344, 344, 344, /* 3220 */ 344, 344, 344, 344, 344, 344, 344, 344, 344, 344, /* 3230 */ 344, 344, 344, 344, 344, 344, 344, 344, 344, 344, /* 3240 */ 344, 344, 344, 344, 344, 344, 344, 344, 344, 344, @@ -1250,93 +1250,93 @@ static const YYCODETYPE yy_lookahead[] = { }; #define YY_SHIFT_COUNT (819) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2265) +#define YY_SHIFT_MAX (2277) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 1329, 0, 237, 0, 475, 475, 475, 475, 475, 475, /* 10 */ 475, 475, 475, 475, 475, 475, 712, 949, 949, 1186, /* 20 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, /* 30 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, /* 40 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, - /* 50 */ 949, 158, 233, 708, 49, 61, 275, 61, 49, 49, - /* 60 */ 61, 1424, 61, 236, 1424, 1424, 180, 61, 46, 80, - /* 70 */ 85, 85, 80, 288, 288, 116, 195, 93, 93, 85, - /* 80 */ 85, 85, 85, 85, 85, 85, 85, 85, 85, 150, - /* 90 */ 85, 85, 292, 46, 85, 85, 472, 46, 85, 85, - /* 100 */ 46, 85, 85, 46, 85, 46, 46, 46, 85, 500, - /* 110 */ 197, 197, 469, 54, 563, 563, 563, 563, 563, 563, - /* 120 */ 563, 563, 563, 563, 563, 563, 563, 563, 563, 563, - /* 130 */ 563, 563, 563, 625, 255, 116, 195, 800, 800, 656, - /* 140 */ 485, 485, 485, 860, 302, 302, 656, 292, 458, 46, - /* 150 */ 46, 338, 46, 620, 46, 620, 620, 646, 713, 840, + /* 50 */ 949, 48, 119, 308, 80, 61, 159, 61, 80, 80, + /* 60 */ 61, 1424, 61, 236, 1424, 1424, 155, 61, 38, 163, + /* 70 */ 56, 56, 163, 490, 490, 49, 232, 52, 52, 56, + /* 80 */ 56, 56, 56, 56, 56, 56, 56, 56, 56, 361, + /* 90 */ 56, 56, 67, 38, 56, 56, 498, 38, 56, 56, + /* 100 */ 38, 56, 56, 38, 56, 38, 38, 38, 56, 460, + /* 110 */ 197, 197, 469, 54, 550, 550, 550, 550, 550, 550, + /* 120 */ 550, 550, 550, 550, 550, 550, 550, 550, 550, 550, + /* 130 */ 550, 550, 550, 894, 255, 49, 232, 639, 639, 503, + /* 140 */ 478, 478, 478, 184, 349, 349, 503, 67, 488, 38, + /* 150 */ 38, 365, 38, 658, 38, 658, 658, 707, 692, 740, /* 160 */ 199, 199, 199, 199, 199, 199, 199, 199, 2197, 434, - /* 170 */ 23, 268, 597, 474, 322, 229, 21, 21, 172, 89, - /* 180 */ 210, 650, 886, 1115, 584, 364, 956, 901, 946, 157, - /* 190 */ 901, 1134, 39, 502, 1031, 1265, 1289, 1328, 1142, 292, - /* 200 */ 1328, 292, 1168, 1344, 1346, 1327, 1355, 1346, 1327, 1207, - /* 210 */ 1344, 1346, 1344, 1327, 1207, 1207, 1286, 1291, 1344, 1299, - /* 220 */ 1344, 1344, 1344, 1385, 1358, 1385, 1358, 1328, 292, 1396, - /* 230 */ 292, 1420, 1422, 292, 1420, 292, 1428, 292, 292, 1344, - /* 240 */ 292, 1385, 46, 46, 46, 46, 46, 46, 46, 46, - /* 250 */ 46, 46, 46, 1344, 840, 840, 1385, 620, 620, 620, - /* 260 */ 1257, 1369, 1328, 500, 1275, 1278, 1396, 500, 1287, 1031, - /* 270 */ 1344, 1355, 1355, 620, 1220, 1226, 620, 1220, 1226, 620, - /* 280 */ 620, 46, 1210, 1317, 1220, 1229, 1227, 1259, 1031, 1224, - /* 290 */ 1243, 1250, 1276, 1346, 1509, 1425, 1277, 1420, 500, 500, - /* 300 */ 1226, 620, 620, 620, 620, 620, 1226, 620, 1389, 500, - /* 310 */ 646, 500, 1346, 1474, 1479, 620, 713, 1344, 500, 1579, - /* 320 */ 1573, 1385, 3215, 3215, 3215, 3215, 3215, 3215, 3215, 3215, - /* 330 */ 3215, 36, 1793, 65, 413, 979, 757, 981, 667, 1019, - /* 340 */ 1092, 833, 741, 1125, 1125, 1125, 1125, 1125, 1125, 1125, - /* 350 */ 1125, 1125, 301, 447, 178, 26, 26, 600, 640, 616, - /* 360 */ 819, 853, 393, 759, 284, 647, 647, 785, 527, 834, - /* 370 */ 785, 785, 785, 1085, 5, 1127, 1169, 1167, 1058, 345, - /* 380 */ 1095, 1104, 1105, 1110, 1188, 1190, 632, 911, 1209, 898, - /* 390 */ 1111, 1172, 1156, 1199, 1203, 1204, 1086, 977, 1025, 1206, - /* 400 */ 1213, 1214, 1216, 1218, 1219, 1240, 1225, 1002, 1236, 1177, - /* 410 */ 1237, 1238, 1242, 1244, 1247, 1248, 1138, 1201, 1208, 1249, - /* 420 */ 1254, 1196, 1272, 1626, 1629, 1631, 1587, 1642, 1608, 1427, - /* 430 */ 1609, 1612, 1614, 1438, 1657, 1623, 1624, 1441, 1627, 1663, - /* 440 */ 1451, 1673, 1637, 1675, 1654, 1678, 1643, 1463, 1681, 1478, - /* 450 */ 1683, 1480, 1481, 1487, 1491, 1688, 1689, 1691, 1500, 1503, - /* 460 */ 1693, 1696, 1550, 1650, 1658, 1709, 1674, 1659, 1712, 1665, - /* 470 */ 1721, 1671, 1717, 1722, 1723, 1676, 1724, 1726, 1727, 1728, - /* 480 */ 1729, 1733, 1570, 1697, 1735, 1575, 1737, 1738, 1739, 1740, - /* 490 */ 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, - /* 500 */ 1752, 1754, 1707, 1757, 1713, 1759, 1760, 1761, 1762, 1763, - /* 510 */ 1764, 1765, 1770, 1618, 1766, 1621, 1771, 1628, 1774, 1784, - /* 520 */ 1768, 1736, 1769, 1756, 1785, 1753, 1767, 1788, 1755, 1797, - /* 530 */ 1758, 1807, 1808, 1773, 1775, 1772, 1811, 1776, 1778, 1783, - /* 540 */ 1812, 1779, 1780, 1790, 1815, 1781, 1820, 1782, 1791, 1801, - /* 550 */ 1786, 1796, 1816, 1798, 1837, 1799, 1804, 1839, 1843, 1848, - /* 560 */ 1809, 1669, 1850, 1786, 1810, 1858, 1861, 1803, 1864, 1865, - /* 570 */ 1835, 1825, 1838, 1877, 1842, 1836, 1847, 1891, 1855, 1845, - /* 580 */ 1851, 1894, 1860, 1852, 1857, 1905, 1906, 1907, 1908, 1909, - /* 590 */ 1910, 1802, 1800, 1874, 1893, 1913, 1880, 1881, 1884, 1885, - /* 600 */ 1886, 1887, 1888, 1889, 1895, 1897, 1896, 1898, 1912, 1899, - /* 610 */ 1931, 1916, 1932, 1917, 1890, 1937, 1918, 1919, 1945, 1946, - /* 620 */ 1953, 1920, 1958, 1922, 1961, 1940, 1943, 1927, 1928, 1929, - /* 630 */ 1866, 1862, 1974, 1805, 1870, 1777, 1949, 1955, 1983, 1789, - /* 640 */ 1965, 1813, 1814, 1991, 1992, 1818, 1817, 1819, 1821, 1993, - /* 650 */ 1966, 1732, 1901, 1903, 1902, 1951, 1914, 1970, 1921, 1904, - /* 660 */ 1972, 1989, 1926, 1930, 1933, 1935, 1936, 2002, 1954, 1976, - /* 670 */ 1938, 2004, 1787, 1939, 1942, 2039, 2012, 1792, 2010, 2013, - /* 680 */ 2014, 2015, 2016, 2017, 1952, 1957, 2006, 1795, 2023, 2011, - /* 690 */ 2065, 2066, 1962, 2025, 1963, 1964, 1967, 1969, 1973, 1892, - /* 700 */ 1975, 2070, 2031, 1924, 1987, 1977, 1786, 2033, 2061, 1994, - /* 710 */ 1846, 1995, 2093, 2075, 1872, 1999, 2000, 2005, 2001, 2007, - /* 720 */ 2003, 2057, 2008, 2009, 2063, 2018, 2095, 1900, 2020, 2019, - /* 730 */ 2022, 2083, 2084, 2021, 2027, 2085, 2026, 2029, 2086, 2032, - /* 740 */ 2030, 2087, 2036, 2037, 2102, 2046, 2038, 2119, 2051, 2040, - /* 750 */ 2042, 2045, 2047, 2053, 2127, 2069, 2149, 2082, 2127, 2127, - /* 760 */ 2167, 2120, 2122, 2153, 2156, 2158, 2160, 2162, 2164, 2165, - /* 770 */ 2166, 2168, 2129, 2104, 2174, 2171, 2172, 2173, 2189, 2176, - /* 780 */ 2177, 2178, 2141, 1895, 2182, 1897, 2183, 2184, 2186, 2188, - /* 790 */ 2200, 2190, 2228, 2192, 2179, 2193, 2231, 2201, 2194, 2195, - /* 800 */ 2234, 2210, 2198, 2208, 2251, 2215, 2202, 2212, 2255, 2223, - /* 810 */ 2227, 2265, 2245, 2235, 2256, 2258, 2259, 2260, 2262, 2257, + /* 170 */ 23, 268, 597, 474, 133, 169, 480, 480, 602, 602, + /* 180 */ 602, 395, 683, 602, 514, 82, 820, 1122, 842, 62, + /* 190 */ 988, 941, 947, 926, 941, 982, 963, 1058, 1063, 1295, + /* 200 */ 1308, 1339, 1152, 67, 1339, 67, 1178, 1354, 1357, 1334, + /* 210 */ 1361, 1357, 1334, 1215, 1354, 1357, 1354, 1334, 1215, 1215, + /* 220 */ 1297, 1301, 1354, 1305, 1354, 1354, 1354, 1391, 1363, 1391, + /* 230 */ 1363, 1339, 67, 1422, 67, 1425, 1427, 67, 1425, 67, + /* 240 */ 1434, 67, 67, 1354, 67, 1391, 38, 38, 38, 38, + /* 250 */ 38, 38, 38, 38, 38, 38, 38, 1354, 740, 740, + /* 260 */ 1391, 658, 658, 658, 1257, 1372, 1339, 460, 1288, 1291, + /* 270 */ 1422, 460, 1298, 1063, 1354, 1361, 1361, 658, 1224, 1230, + /* 280 */ 1224, 1230, 38, 1223, 1321, 1224, 1229, 1231, 1259, 1063, + /* 290 */ 1240, 1255, 1258, 1265, 1357, 1519, 1431, 1275, 1425, 460, + /* 300 */ 460, 1230, 658, 658, 658, 658, 1230, 658, 1399, 460, + /* 310 */ 707, 460, 1357, 1479, 1480, 658, 692, 1354, 460, 1579, + /* 320 */ 1576, 1391, 3215, 3215, 3215, 3215, 3215, 3215, 3215, 3215, + /* 330 */ 3215, 36, 1793, 65, 413, 726, 750, 856, 666, 779, + /* 340 */ 979, 833, 741, 1086, 1086, 1086, 1086, 1086, 1086, 1086, + /* 350 */ 1086, 1086, 79, 447, 424, 322, 322, 241, 356, 860, + /* 360 */ 706, 116, 869, 1017, 717, 908, 908, 757, 632, 186, + /* 370 */ 757, 757, 757, 410, 1126, 1111, 467, 1072, 769, 1113, + /* 380 */ 1098, 1104, 1105, 1110, 1115, 1188, 293, 1209, 1220, 1012, + /* 390 */ 1127, 1142, 1163, 1203, 1204, 1206, 1102, 968, 1195, 1208, + /* 400 */ 1213, 1214, 1218, 1219, 1225, 1252, 1236, 294, 1237, 1180, + /* 410 */ 1242, 1244, 1246, 1247, 1248, 1249, 324, 1201, 1250, 1227, + /* 420 */ 1254, 1196, 890, 1631, 1632, 1634, 1602, 1648, 1612, 1430, + /* 430 */ 1620, 1622, 1623, 1440, 1662, 1626, 1627, 1451, 1636, 1674, + /* 440 */ 1454, 1676, 1641, 1679, 1658, 1681, 1645, 1467, 1684, 1481, + /* 450 */ 1686, 1483, 1484, 1490, 1495, 1692, 1693, 1694, 1504, 1506, + /* 460 */ 1699, 1707, 1562, 1661, 1663, 1711, 1677, 1665, 1717, 1672, + /* 470 */ 1722, 1678, 1724, 1725, 1726, 1680, 1727, 1728, 1733, 1734, + /* 480 */ 1735, 1736, 1570, 1700, 1738, 1578, 1740, 1742, 1743, 1744, + /* 490 */ 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1752, 1754, 1756, + /* 500 */ 1757, 1758, 1710, 1760, 1718, 1761, 1762, 1764, 1765, 1766, + /* 510 */ 1767, 1763, 1770, 1624, 1773, 1628, 1784, 1642, 1786, 1788, + /* 520 */ 1768, 1741, 1775, 1769, 1804, 1753, 1771, 1806, 1778, 1807, + /* 530 */ 1779, 1810, 1811, 1776, 1774, 1772, 1812, 1781, 1780, 1785, + /* 540 */ 1815, 1783, 1782, 1787, 1816, 1789, 1822, 1790, 1792, 1797, + /* 550 */ 1796, 1798, 1818, 1799, 1837, 1803, 1808, 1839, 1851, 1856, + /* 560 */ 1817, 1671, 1861, 1796, 1823, 1864, 1865, 1805, 1874, 1877, + /* 570 */ 1842, 1829, 1845, 1889, 1854, 1841, 1852, 1893, 1859, 1846, + /* 580 */ 1857, 1903, 1868, 1855, 1866, 1907, 1909, 1910, 1911, 1912, + /* 590 */ 1913, 1801, 1813, 1879, 1895, 1918, 1882, 1885, 1886, 1887, + /* 600 */ 1891, 1894, 1896, 1897, 1899, 1902, 1900, 1901, 1904, 1906, + /* 610 */ 1930, 1914, 1939, 1923, 1905, 1940, 1924, 1916, 1956, 1957, + /* 620 */ 1959, 1925, 1961, 1926, 1964, 1943, 1946, 1931, 1936, 1937, + /* 630 */ 1869, 1871, 1980, 1819, 1875, 1791, 1949, 1965, 1983, 1800, + /* 640 */ 1966, 1820, 1814, 1989, 1991, 1825, 1809, 1826, 1821, 1997, + /* 650 */ 1970, 1737, 1917, 1898, 1919, 1958, 1929, 1960, 1920, 1928, + /* 660 */ 1978, 1985, 1932, 1935, 1938, 1941, 1942, 1987, 1988, 1993, + /* 670 */ 1944, 2003, 1795, 1945, 1947, 2040, 2012, 1802, 2009, 2014, + /* 680 */ 2016, 2018, 2019, 2020, 1951, 1952, 2011, 1830, 2029, 2015, + /* 690 */ 2066, 2067, 1962, 2027, 1967, 1963, 1968, 1973, 1975, 1922, + /* 700 */ 1976, 2071, 2032, 1921, 1990, 1977, 1796, 2045, 2064, 1994, + /* 710 */ 1850, 2001, 2093, 2084, 1880, 2004, 2000, 2005, 2006, 2008, + /* 720 */ 2010, 2063, 2017, 2021, 2072, 2013, 2100, 1908, 2022, 1999, + /* 730 */ 2023, 2087, 2088, 2026, 2028, 2089, 2030, 2035, 2097, 2031, + /* 740 */ 2037, 2102, 2039, 2049, 2115, 2051, 2052, 2123, 2061, 2042, + /* 750 */ 2047, 2048, 2059, 2082, 2139, 2083, 2153, 2085, 2139, 2139, + /* 760 */ 2170, 2122, 2125, 2160, 2162, 2163, 2165, 2166, 2167, 2168, + /* 770 */ 2169, 2171, 2130, 2109, 2177, 2174, 2176, 2178, 2192, 2181, + /* 780 */ 2182, 2183, 2144, 1899, 2186, 1902, 2188, 2190, 2191, 2193, + /* 790 */ 2200, 2194, 2229, 2198, 2187, 2195, 2234, 2208, 2196, 2207, + /* 800 */ 2250, 2214, 2201, 2211, 2254, 2223, 2204, 2222, 2265, 2230, + /* 810 */ 2231, 2277, 2256, 2246, 2258, 2260, 2261, 2262, 2264, 2266, }; #define YY_REDUCE_COUNT (330) -#define YY_REDUCE_MIN (-467) +#define YY_REDUCE_MIN (-468) #define YY_REDUCE_MAX (2762) static const short yy_reduce_ofst[] = { /* 0 */ -271, -301, -92, 144, 235, 380, 450, 473, 618, 709, @@ -1344,119 +1344,119 @@ static const short yy_reduce_ofst[] = { /* 20 */ 1502, 1567, 1582, 1638, 1701, 1716, 1731, 1794, 1824, 1915, /* 30 */ 1948, 1971, 1986, 2062, 2077, 2092, 2155, 2185, 2218, 2296, /* 40 */ 2332, 2361, 2426, 2493, 2509, 2574, 2603, 2668, 2697, 2733, - /* 50 */ 2762, -303, 128, -123, 104, 204, 508, 624, 175, 398, - /* 60 */ 679, 683, -467, -344, -302, 587, -465, -295, -377, -230, - /* 70 */ -358, -206, -151, -286, -150, -251, -103, -46, 55, 88, - /* 80 */ 130, 177, 198, 16, 207, 262, 269, 309, 331, 102, - /* 90 */ 369, 384, 153, -110, 415, 419, -343, -91, 457, 525, - /* 100 */ -380, 643, 657, 250, 665, 546, 342, 658, 682, 248, - /* 110 */ 119, 119, -117, 114, 2, 17, 164, 279, 286, 289, - /* 120 */ 440, 516, 581, 591, 619, 645, 693, 703, 727, 729, - /* 130 */ 742, 743, 752, -204, -368, -109, 422, 507, 526, 594, - /* 140 */ -368, -38, 323, 443, 355, 395, 601, 427, -374, 343, - /* 150 */ 582, 553, 167, 615, 725, 651, 717, 738, 771, 728, - /* 160 */ 479, 512, 528, 575, 583, 660, 664, 583, 689, 863, - /* 170 */ 732, 841, 756, 770, 897, 778, 890, 892, 894, 852, - /* 180 */ 894, 915, 871, 927, 933, 896, 874, 823, 823, 805, - /* 190 */ 823, 843, 846, 894, 885, 876, 906, 917, 918, 989, - /* 200 */ 925, 991, 937, 1006, 1011, 965, 963, 1017, 969, 973, - /* 210 */ 1022, 1026, 1028, 976, 982, 983, 1020, 1027, 1032, 1033, - /* 220 */ 1042, 1043, 1045, 1055, 1053, 1059, 1054, 980, 1044, 1036, - /* 230 */ 1071, 1080, 1018, 1076, 1084, 1079, 1034, 1081, 1083, 1093, - /* 240 */ 1087, 1101, 1066, 1068, 1069, 1070, 1072, 1074, 1075, 1077, - /* 250 */ 1078, 1082, 1088, 1108, 1114, 1118, 1117, 1089, 1090, 1091, - /* 260 */ 1037, 1046, 1047, 1112, 1050, 1052, 1094, 1119, 1056, 1073, - /* 270 */ 1133, 1096, 1097, 1098, 1012, 1103, 1099, 1021, 1107, 1102, - /* 280 */ 1106, 894, 1023, 1029, 1041, 1024, 1035, 1051, 1109, 1030, - /* 290 */ 1039, 1048, 823, 1179, 1100, 1113, 1120, 1184, 1181, 1192, - /* 300 */ 1132, 1157, 1159, 1161, 1164, 1165, 1149, 1175, 1163, 1212, - /* 310 */ 1198, 1223, 1232, 1122, 1194, 1197, 1222, 1239, 1230, 1255, - /* 320 */ 1260, 1262, 1200, 1187, 1215, 1228, 1233, 1246, 1253, 1251, - /* 330 */ 1280, + /* 50 */ 2762, -303, 128, 148, 196, 505, 543, 545, 579, 665, + /* 60 */ 587, 47, -468, -344, -302, -91, -466, -455, -362, -241, + /* 70 */ -358, -187, -151, -347, -150, -115, -25, -168, -13, -81, + /* 80 */ 57, 142, 269, -69, 207, 272, 310, 378, 283, -47, + /* 90 */ 457, 524, -323, 182, 652, 714, 191, 298, 727, 748, + /* 100 */ 348, 756, 759, 302, 762, 387, 463, 594, 783, 430, + /* 110 */ -421, -421, -360, -86, -240, 96, 153, 164, 216, 300, + /* 120 */ 329, 516, 564, 570, 577, 628, 653, 725, 729, 749, + /* 130 */ 763, 765, 767, 344, -38, -10, 156, 435, 444, 531, + /* 140 */ -38, 354, 529, 270, 527, 576, 649, 392, -206, 256, + /* 150 */ 700, 217, -283, 662, 701, 753, 774, 795, 825, 789, + /* 160 */ -389, -385, 485, 512, 542, 694, 870, 542, 661, 883, + /* 170 */ 913, 849, 764, 776, 905, 791, 896, 902, 794, 796, + /* 180 */ 804, 907, 862, 807, 907, 935, 886, 948, 956, 909, + /* 190 */ 904, 836, 836, 828, 836, 858, 848, 907, 921, 916, + /* 200 */ 918, 929, 927, 996, 933, 1003, 950, 1016, 1018, 971, + /* 210 */ 972, 1026, 978, 983, 1031, 1032, 1034, 989, 985, 994, + /* 220 */ 1030, 1033, 1046, 1037, 1049, 1050, 1051, 1060, 1082, 1064, + /* 230 */ 1083, 1007, 1073, 1041, 1076, 1088, 1025, 1084, 1090, 1085, + /* 240 */ 1039, 1089, 1092, 1097, 1094, 1106, 1075, 1077, 1078, 1079, + /* 250 */ 1080, 1081, 1087, 1091, 1093, 1095, 1099, 1117, 1114, 1120, + /* 260 */ 1125, 1074, 1096, 1100, 1043, 1047, 1053, 1123, 1059, 1062, + /* 270 */ 1101, 1128, 1065, 1107, 1138, 1103, 1108, 1109, 1020, 1112, + /* 280 */ 1023, 1116, 907, 1028, 1027, 1055, 1029, 1045, 1044, 1118, + /* 290 */ 1038, 1054, 1048, 836, 1173, 1121, 1119, 1129, 1197, 1193, + /* 300 */ 1198, 1139, 1161, 1167, 1168, 1175, 1159, 1182, 1166, 1226, + /* 310 */ 1200, 1232, 1235, 1135, 1221, 1189, 1212, 1241, 1245, 1266, + /* 320 */ 1268, 1273, 1199, 1184, 1210, 1216, 1260, 1263, 1264, 1261, + /* 330 */ 1282, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 10 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 20 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 30 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 40 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 50 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 60 */ 2164, 1835, 1835, 2127, 1835, 1835, 1835, 1835, 1835, 1835, - /* 70 */ 1835, 1835, 1835, 1835, 1835, 2134, 1835, 1835, 1835, 1835, - /* 80 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 90 */ 1835, 1835, 1931, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 100 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1929, - /* 110 */ 2358, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 120 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 130 */ 1835, 1835, 1835, 1835, 2370, 1835, 1835, 1905, 1905, 1835, - /* 140 */ 2370, 2370, 2370, 1929, 2330, 2330, 1835, 1931, 2198, 1835, - /* 150 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 2053, 1835, 1865, - /* 160 */ 1835, 1835, 1835, 1835, 2077, 1835, 1835, 1835, 2190, 1835, - /* 170 */ 1835, 2399, 2456, 1835, 1835, 2402, 1835, 1835, 1835, 1835, - /* 180 */ 1835, 1835, 2139, 1835, 1835, 2006, 2389, 2362, 2376, 2440, - /* 190 */ 2363, 2360, 2383, 1835, 2393, 1835, 2222, 1835, 2212, 1931, - /* 200 */ 1835, 1931, 2177, 2122, 1835, 2132, 1835, 1835, 2132, 2129, - /* 210 */ 1835, 1835, 1835, 2132, 2129, 2129, 1995, 1991, 1835, 1989, - /* 220 */ 1835, 1835, 1835, 1835, 1889, 1835, 1889, 1835, 1931, 1835, - /* 230 */ 1931, 1835, 1835, 1931, 1835, 1931, 1835, 1931, 1931, 1835, - /* 240 */ 1931, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 250 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 260 */ 2210, 2196, 1835, 1929, 2188, 2186, 1835, 1929, 2184, 2393, - /* 270 */ 1835, 1835, 1835, 1835, 2410, 2408, 1835, 2410, 2408, 1835, - /* 280 */ 1835, 1835, 2424, 2420, 2410, 2429, 2426, 2395, 2393, 2459, - /* 290 */ 2446, 2442, 2376, 1835, 1835, 2381, 2379, 1835, 1929, 1929, - /* 300 */ 2408, 1835, 1835, 1835, 1835, 1835, 2408, 1835, 1835, 1929, - /* 310 */ 1835, 1929, 1835, 1835, 2022, 1835, 1835, 1835, 1929, 1835, - /* 320 */ 1874, 1835, 2179, 2201, 2160, 2160, 2056, 2056, 2056, 1932, - /* 330 */ 1840, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 340 */ 1835, 1835, 1835, 2423, 2422, 2285, 1835, 2334, 2333, 2332, - /* 350 */ 2323, 2284, 2018, 1835, 1835, 2283, 2282, 1835, 1835, 1835, - /* 360 */ 1835, 1835, 1835, 1835, 1835, 2151, 2150, 2276, 1835, 1835, - /* 370 */ 2277, 2275, 2274, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 380 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 390 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 2443, 2447, 1835, - /* 400 */ 1835, 1835, 1835, 1835, 1835, 2359, 1835, 1835, 1835, 2258, - /* 410 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 420 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 430 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 440 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 450 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 460 */ 1835, 1835, 2128, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 470 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 480 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 490 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 500 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 510 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 2143, 1835, 1835, - /* 520 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 530 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 540 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1879, - /* 550 */ 2263, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 560 */ 1835, 1835, 1835, 2266, 1835, 1835, 1835, 1835, 1835, 1835, - /* 570 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 580 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 590 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 600 */ 1835, 1835, 1835, 1835, 1971, 1970, 1835, 1835, 1835, 1835, - /* 610 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 620 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 630 */ 2267, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 640 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 2439, - /* 650 */ 2396, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 660 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 2258, - /* 670 */ 1835, 2421, 1835, 1835, 2437, 1835, 2441, 1835, 1835, 1835, - /* 680 */ 1835, 1835, 1835, 1835, 2369, 2365, 1835, 1835, 2361, 1835, - /* 690 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 700 */ 1835, 1835, 1835, 1835, 1835, 1835, 2257, 1835, 2320, 1835, - /* 710 */ 1835, 1835, 2354, 1835, 1835, 2305, 1835, 1835, 1835, 1835, - /* 720 */ 1835, 1835, 1835, 1835, 1835, 2267, 1835, 2270, 1835, 1835, - /* 730 */ 1835, 1835, 1835, 2050, 1835, 1835, 1835, 1835, 1835, 1835, - /* 740 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 2034, - /* 750 */ 2032, 2031, 2030, 1835, 2063, 1835, 1835, 1835, 2059, 2058, - /* 760 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 770 */ 1835, 1835, 1835, 1835, 1950, 1835, 1835, 1835, 1835, 1835, - /* 780 */ 1835, 1835, 1835, 1942, 1835, 1941, 1835, 1835, 1835, 1835, - /* 790 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 800 */ 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, 1835, - /* 810 */ 1835, 1835, 1835, 1864, 1835, 1835, 1835, 1835, 1835, 1835, + /* 0 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 10 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 20 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 30 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 40 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 50 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 60 */ 2170, 1841, 1841, 2133, 1841, 1841, 1841, 1841, 1841, 1841, + /* 70 */ 1841, 1841, 1841, 1841, 1841, 2140, 1841, 1841, 1841, 1841, + /* 80 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 90 */ 1841, 1841, 1937, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 100 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1935, + /* 110 */ 2364, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 120 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 130 */ 1841, 1841, 1841, 1841, 2376, 1841, 1841, 1911, 1911, 1841, + /* 140 */ 2376, 2376, 2376, 1935, 2336, 2336, 1841, 1937, 2204, 1841, + /* 150 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 2059, 1841, 1871, + /* 160 */ 1841, 1841, 1841, 1841, 2083, 1841, 1841, 1841, 2196, 1841, + /* 170 */ 1841, 2405, 2465, 1841, 1841, 2408, 1841, 1841, 1841, 1841, + /* 180 */ 1841, 1841, 1841, 1841, 1841, 1841, 2145, 1841, 1841, 2012, + /* 190 */ 2395, 2368, 2382, 2449, 2369, 2366, 2389, 1841, 2399, 1841, + /* 200 */ 2228, 1841, 2218, 1937, 1841, 1937, 2183, 2128, 1841, 2138, + /* 210 */ 1841, 1841, 2138, 2135, 1841, 1841, 1841, 2138, 2135, 2135, + /* 220 */ 2001, 1997, 1841, 1995, 1841, 1841, 1841, 1841, 1895, 1841, + /* 230 */ 1895, 1841, 1937, 1841, 1937, 1841, 1841, 1937, 1841, 1937, + /* 240 */ 1841, 1937, 1937, 1841, 1937, 1841, 1841, 1841, 1841, 1841, + /* 250 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 260 */ 1841, 1841, 1841, 1841, 2216, 2202, 1841, 1935, 2194, 2192, + /* 270 */ 1841, 1935, 2190, 2399, 1841, 1841, 1841, 1841, 2419, 2414, + /* 280 */ 2419, 2414, 1841, 2433, 2429, 2419, 2438, 2435, 2401, 2399, + /* 290 */ 2468, 2455, 2451, 2382, 1841, 1841, 2387, 2385, 1841, 1935, + /* 300 */ 1935, 2414, 1841, 1841, 1841, 1841, 2414, 1841, 1841, 1935, + /* 310 */ 1841, 1935, 1841, 1841, 2028, 1841, 1841, 1841, 1935, 1841, + /* 320 */ 1880, 1841, 2185, 2207, 2166, 2166, 2062, 2062, 2062, 1938, + /* 330 */ 1846, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 340 */ 1841, 1841, 1841, 2432, 2431, 2291, 1841, 2340, 2339, 2338, + /* 350 */ 2329, 2290, 2024, 1841, 1841, 2289, 2288, 1841, 1841, 1841, + /* 360 */ 1841, 1841, 1841, 1841, 1841, 2157, 2156, 2282, 1841, 1841, + /* 370 */ 2283, 2281, 2280, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 380 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 390 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 2452, 2456, 1841, + /* 400 */ 1841, 1841, 1841, 1841, 1841, 2365, 1841, 1841, 1841, 2264, + /* 410 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 420 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 430 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 440 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 450 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 460 */ 1841, 1841, 2134, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 470 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 480 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 490 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 500 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 510 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 2149, 1841, 1841, + /* 520 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 530 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 540 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1885, + /* 550 */ 2269, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 560 */ 1841, 1841, 1841, 2272, 1841, 1841, 1841, 1841, 1841, 1841, + /* 570 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 580 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 590 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 600 */ 1841, 1841, 1841, 1841, 1977, 1976, 1841, 1841, 1841, 1841, + /* 610 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 620 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 630 */ 2273, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 640 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 2448, + /* 650 */ 2402, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 660 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 2264, + /* 670 */ 1841, 2430, 1841, 1841, 2446, 1841, 2450, 1841, 1841, 1841, + /* 680 */ 1841, 1841, 1841, 1841, 2375, 2371, 1841, 1841, 2367, 1841, + /* 690 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 700 */ 1841, 1841, 1841, 1841, 1841, 1841, 2263, 1841, 2326, 1841, + /* 710 */ 1841, 1841, 2360, 1841, 1841, 2311, 1841, 1841, 1841, 1841, + /* 720 */ 1841, 1841, 1841, 1841, 1841, 2273, 1841, 2276, 1841, 1841, + /* 730 */ 1841, 1841, 1841, 2056, 1841, 1841, 1841, 1841, 1841, 1841, + /* 740 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 2040, + /* 750 */ 2038, 2037, 2036, 1841, 2069, 1841, 1841, 1841, 2065, 2064, + /* 760 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 770 */ 1841, 1841, 1841, 1841, 1956, 1841, 1841, 1841, 1841, 1841, + /* 780 */ 1841, 1841, 1841, 1948, 1841, 1947, 1841, 1841, 1841, 1841, + /* 790 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 800 */ 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, 1841, + /* 810 */ 1841, 1841, 1841, 1870, 1841, 1841, 1841, 1841, 1841, 1841, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1871,9 +1871,9 @@ struct yyParser { }; typedef struct yyParser yyParser; -#include #ifndef NDEBUG #include +#include static FILE *yyTraceFILE = 0; static char *yyTracePrompt = 0; #endif /* NDEBUG */ @@ -2397,19 +2397,20 @@ static const char *const yyTokenName[] = { /* 486 */ "select_item", /* 487 */ "partition_list", /* 488 */ "partition_item", - /* 489 */ "fill_mode", - /* 490 */ "group_by_list", - /* 491 */ "query_expression", - /* 492 */ "query_simple", - /* 493 */ "order_by_clause_opt", - /* 494 */ "slimit_clause_opt", - /* 495 */ "limit_clause_opt", - /* 496 */ "union_query_expression", - /* 497 */ "query_simple_or_subquery", - /* 498 */ "sort_specification_list", - /* 499 */ "sort_specification", - /* 500 */ "ordering_specification_opt", - /* 501 */ "null_ordering_opt", + /* 489 */ "interval_sliding_duration_literal", + /* 490 */ "fill_mode", + /* 491 */ "group_by_list", + /* 492 */ "query_expression", + /* 493 */ "query_simple", + /* 494 */ "order_by_clause_opt", + /* 495 */ "slimit_clause_opt", + /* 496 */ "limit_clause_opt", + /* 497 */ "union_query_expression", + /* 498 */ "query_simple_or_subquery", + /* 499 */ "sort_specification_list", + /* 500 */ "sort_specification", + /* 501 */ "ordering_specification_opt", + /* 502 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -2982,65 +2983,68 @@ static const char *const yyRuleName[] = { /* 562 */ "partition_item ::= expr_or_subquery column_alias", /* 563 */ "partition_item ::= expr_or_subquery AS column_alias", /* 564 */ "twindow_clause_opt ::=", - /* 565 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 565 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA interval_sliding_duration_literal NK_RP", /* 566 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 567 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 568 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 567 */ "twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_RP sliding_opt fill_opt", + /* 568 */ "twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_COMMA interval_sliding_duration_literal NK_RP sliding_opt fill_opt", /* 569 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", /* 570 */ "sliding_opt ::=", - /* 571 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 572 */ "fill_opt ::=", - /* 573 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 574 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", - /* 575 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", - /* 576 */ "fill_mode ::= NONE", - /* 577 */ "fill_mode ::= PREV", - /* 578 */ "fill_mode ::= NULL", - /* 579 */ "fill_mode ::= NULL_F", - /* 580 */ "fill_mode ::= LINEAR", - /* 581 */ "fill_mode ::= NEXT", - /* 582 */ "group_by_clause_opt ::=", - /* 583 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 584 */ "group_by_list ::= expr_or_subquery", - /* 585 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 586 */ "having_clause_opt ::=", - /* 587 */ "having_clause_opt ::= HAVING search_condition", - /* 588 */ "range_opt ::=", - /* 589 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 590 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_RP", - /* 591 */ "every_opt ::=", - /* 592 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 593 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 594 */ "query_simple ::= query_specification", - /* 595 */ "query_simple ::= union_query_expression", - /* 596 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 597 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 598 */ "query_simple_or_subquery ::= query_simple", - /* 599 */ "query_simple_or_subquery ::= subquery", - /* 600 */ "query_or_subquery ::= query_expression", - /* 601 */ "query_or_subquery ::= subquery", - /* 602 */ "order_by_clause_opt ::=", - /* 603 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 604 */ "slimit_clause_opt ::=", - /* 605 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 606 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 607 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 608 */ "limit_clause_opt ::=", - /* 609 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 610 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 611 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 612 */ "subquery ::= NK_LP query_expression NK_RP", - /* 613 */ "subquery ::= NK_LP subquery NK_RP", - /* 614 */ "search_condition ::= common_expression", - /* 615 */ "sort_specification_list ::= sort_specification", - /* 616 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 617 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 618 */ "ordering_specification_opt ::=", - /* 619 */ "ordering_specification_opt ::= ASC", - /* 620 */ "ordering_specification_opt ::= DESC", - /* 621 */ "null_ordering_opt ::=", - /* 622 */ "null_ordering_opt ::= NULLS FIRST", - /* 623 */ "null_ordering_opt ::= NULLS LAST", + /* 571 */ "sliding_opt ::= SLIDING NK_LP interval_sliding_duration_literal NK_RP", + /* 572 */ "interval_sliding_duration_literal ::= NK_VARIABLE", + /* 573 */ "interval_sliding_duration_literal ::= NK_STRING", + /* 574 */ "interval_sliding_duration_literal ::= NK_INTEGER", + /* 575 */ "fill_opt ::=", + /* 576 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 577 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 578 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", + /* 579 */ "fill_mode ::= NONE", + /* 580 */ "fill_mode ::= PREV", + /* 581 */ "fill_mode ::= NULL", + /* 582 */ "fill_mode ::= NULL_F", + /* 583 */ "fill_mode ::= LINEAR", + /* 584 */ "fill_mode ::= NEXT", + /* 585 */ "group_by_clause_opt ::=", + /* 586 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 587 */ "group_by_list ::= expr_or_subquery", + /* 588 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 589 */ "having_clause_opt ::=", + /* 590 */ "having_clause_opt ::= HAVING search_condition", + /* 591 */ "range_opt ::=", + /* 592 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 593 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_RP", + /* 594 */ "every_opt ::=", + /* 595 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 596 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 597 */ "query_simple ::= query_specification", + /* 598 */ "query_simple ::= union_query_expression", + /* 599 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 600 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 601 */ "query_simple_or_subquery ::= query_simple", + /* 602 */ "query_simple_or_subquery ::= subquery", + /* 603 */ "query_or_subquery ::= query_expression", + /* 604 */ "query_or_subquery ::= subquery", + /* 605 */ "order_by_clause_opt ::=", + /* 606 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 607 */ "slimit_clause_opt ::=", + /* 608 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 609 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 610 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 611 */ "limit_clause_opt ::=", + /* 612 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 613 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 614 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 615 */ "subquery ::= NK_LP query_expression NK_RP", + /* 616 */ "subquery ::= NK_LP subquery NK_RP", + /* 617 */ "search_condition ::= common_expression", + /* 618 */ "sort_specification_list ::= sort_specification", + /* 619 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 620 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 621 */ "ordering_specification_opt ::=", + /* 622 */ "ordering_specification_opt ::= ASC", + /* 623 */ "ordering_specification_opt ::= DESC", + /* 624 */ "null_ordering_opt ::=", + /* 625 */ "null_ordering_opt ::= NULLS FIRST", + /* 626 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -3234,17 +3238,18 @@ static void yy_destructor( case 485: /* having_clause_opt */ case 486: /* select_item */ case 488: /* partition_item */ - case 491: /* query_expression */ - case 492: /* query_simple */ - case 494: /* slimit_clause_opt */ - case 495: /* limit_clause_opt */ - case 496: /* union_query_expression */ - case 497: /* query_simple_or_subquery */ - case 499: /* sort_specification */ + case 489: /* interval_sliding_duration_literal */ + case 492: /* query_expression */ + case 493: /* query_simple */ + case 495: /* slimit_clause_opt */ + case 496: /* limit_clause_opt */ + case 497: /* union_query_expression */ + case 498: /* query_simple_or_subquery */ + case 500: /* sort_specification */ { #line 7 "sql.y" - nodesDestroyNode((yypminor->yy348)); -#line 3247 "sql.c" + nodesDestroyNode((yypminor->yy342)); +#line 3252 "sql.c" } break; case 345: /* account_options */ @@ -3256,7 +3261,7 @@ static void yy_destructor( { #line 54 "sql.y" -#line 3259 "sql.c" +#line 3264 "sql.c" } break; case 349: /* ip_range_list */ @@ -3289,13 +3294,13 @@ static void yy_destructor( case 479: /* partition_by_clause_opt */ case 484: /* group_by_clause_opt */ case 487: /* partition_list */ - case 490: /* group_by_list */ - case 493: /* order_by_clause_opt */ - case 498: /* sort_specification_list */ + case 491: /* group_by_list */ + case 494: /* order_by_clause_opt */ + case 499: /* sort_specification_list */ { #line 85 "sql.y" - nodesDestroyList((yypminor->yy860)); -#line 3298 "sql.c" + nodesDestroyList((yypminor->yy20)); +#line 3303 "sql.c" } break; case 352: /* user_name */ @@ -3319,14 +3324,14 @@ static void yy_destructor( { #line 792 "sql.y" -#line 3322 "sql.c" +#line 3327 "sql.c" } break; case 353: /* sysinfo_opt */ { #line 112 "sql.y" -#line 3329 "sql.c" +#line 3334 "sql.c" } break; case 354: /* privileges */ @@ -3335,14 +3340,14 @@ static void yy_destructor( { #line 121 "sql.y" -#line 3338 "sql.c" +#line 3343 "sql.c" } break; case 355: /* priv_level */ { #line 137 "sql.y" -#line 3345 "sql.c" +#line 3350 "sql.c" } break; case 364: /* force_opt */ @@ -3358,7 +3363,7 @@ static void yy_destructor( { #line 166 "sql.y" -#line 3361 "sql.c" +#line 3366 "sql.c" } break; case 377: /* alter_db_option */ @@ -3366,14 +3371,14 @@ static void yy_destructor( { #line 264 "sql.y" -#line 3369 "sql.c" +#line 3374 "sql.c" } break; case 389: /* type_name */ { #line 386 "sql.y" -#line 3376 "sql.c" +#line 3381 "sql.c" } break; case 404: /* db_kind_opt */ @@ -3381,14 +3386,14 @@ static void yy_destructor( { #line 547 "sql.y" -#line 3384 "sql.c" +#line 3389 "sql.c" } break; case 405: /* table_kind_db_name_cond_opt */ { #line 512 "sql.y" -#line 3391 "sql.c" +#line 3396 "sql.c" } break; case 460: /* compare_op */ @@ -3396,35 +3401,35 @@ static void yy_destructor( { #line 980 "sql.y" -#line 3399 "sql.c" +#line 3404 "sql.c" } break; case 473: /* join_type */ { #line 1056 "sql.y" -#line 3406 "sql.c" +#line 3411 "sql.c" } break; - case 489: /* fill_mode */ + case 490: /* fill_mode */ { -#line 1142 "sql.y" +#line 1147 "sql.y" -#line 3413 "sql.c" +#line 3418 "sql.c" } break; - case 500: /* ordering_specification_opt */ + case 501: /* ordering_specification_opt */ { -#line 1227 "sql.y" +#line 1232 "sql.y" -#line 3420 "sql.c" +#line 3425 "sql.c" } break; - case 501: /* null_ordering_opt */ + case 502: /* null_ordering_opt */ { -#line 1233 "sql.y" +#line 1238 "sql.y" -#line 3427 "sql.c" +#line 3432 "sql.c" } break; /********* End destructor definitions *****************************************/ @@ -4278,65 +4283,68 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 488, /* (562) partition_item ::= expr_or_subquery column_alias */ 488, /* (563) partition_item ::= expr_or_subquery AS column_alias */ 483, /* (564) twindow_clause_opt ::= */ - 483, /* (565) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + 483, /* (565) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA interval_sliding_duration_literal NK_RP */ 483, /* (566) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - 483, /* (567) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - 483, /* (568) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + 483, /* (567) twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ + 483, /* (568) twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_COMMA interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ 483, /* (569) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ 418, /* (570) sliding_opt ::= */ - 418, /* (571) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - 482, /* (572) fill_opt ::= */ - 482, /* (573) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - 482, /* (574) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ - 482, /* (575) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ - 489, /* (576) fill_mode ::= NONE */ - 489, /* (577) fill_mode ::= PREV */ - 489, /* (578) fill_mode ::= NULL */ - 489, /* (579) fill_mode ::= NULL_F */ - 489, /* (580) fill_mode ::= LINEAR */ - 489, /* (581) fill_mode ::= NEXT */ - 484, /* (582) group_by_clause_opt ::= */ - 484, /* (583) group_by_clause_opt ::= GROUP BY group_by_list */ - 490, /* (584) group_by_list ::= expr_or_subquery */ - 490, /* (585) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - 485, /* (586) having_clause_opt ::= */ - 485, /* (587) having_clause_opt ::= HAVING search_condition */ - 480, /* (588) range_opt ::= */ - 480, /* (589) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - 480, /* (590) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ - 481, /* (591) every_opt ::= */ - 481, /* (592) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - 491, /* (593) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - 492, /* (594) query_simple ::= query_specification */ - 492, /* (595) query_simple ::= union_query_expression */ - 496, /* (596) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - 496, /* (597) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - 497, /* (598) query_simple_or_subquery ::= query_simple */ - 497, /* (599) query_simple_or_subquery ::= subquery */ - 423, /* (600) query_or_subquery ::= query_expression */ - 423, /* (601) query_or_subquery ::= subquery */ - 493, /* (602) order_by_clause_opt ::= */ - 493, /* (603) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 494, /* (604) slimit_clause_opt ::= */ - 494, /* (605) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - 494, /* (606) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - 494, /* (607) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 495, /* (608) limit_clause_opt ::= */ - 495, /* (609) limit_clause_opt ::= LIMIT NK_INTEGER */ - 495, /* (610) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - 495, /* (611) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 471, /* (612) subquery ::= NK_LP query_expression NK_RP */ - 471, /* (613) subquery ::= NK_LP subquery NK_RP */ - 362, /* (614) search_condition ::= common_expression */ - 498, /* (615) sort_specification_list ::= sort_specification */ - 498, /* (616) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - 499, /* (617) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 500, /* (618) ordering_specification_opt ::= */ - 500, /* (619) ordering_specification_opt ::= ASC */ - 500, /* (620) ordering_specification_opt ::= DESC */ - 501, /* (621) null_ordering_opt ::= */ - 501, /* (622) null_ordering_opt ::= NULLS FIRST */ - 501, /* (623) null_ordering_opt ::= NULLS LAST */ + 418, /* (571) sliding_opt ::= SLIDING NK_LP interval_sliding_duration_literal NK_RP */ + 489, /* (572) interval_sliding_duration_literal ::= NK_VARIABLE */ + 489, /* (573) interval_sliding_duration_literal ::= NK_STRING */ + 489, /* (574) interval_sliding_duration_literal ::= NK_INTEGER */ + 482, /* (575) fill_opt ::= */ + 482, /* (576) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + 482, /* (577) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + 482, /* (578) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + 490, /* (579) fill_mode ::= NONE */ + 490, /* (580) fill_mode ::= PREV */ + 490, /* (581) fill_mode ::= NULL */ + 490, /* (582) fill_mode ::= NULL_F */ + 490, /* (583) fill_mode ::= LINEAR */ + 490, /* (584) fill_mode ::= NEXT */ + 484, /* (585) group_by_clause_opt ::= */ + 484, /* (586) group_by_clause_opt ::= GROUP BY group_by_list */ + 491, /* (587) group_by_list ::= expr_or_subquery */ + 491, /* (588) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + 485, /* (589) having_clause_opt ::= */ + 485, /* (590) having_clause_opt ::= HAVING search_condition */ + 480, /* (591) range_opt ::= */ + 480, /* (592) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + 480, /* (593) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 481, /* (594) every_opt ::= */ + 481, /* (595) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + 492, /* (596) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + 493, /* (597) query_simple ::= query_specification */ + 493, /* (598) query_simple ::= union_query_expression */ + 497, /* (599) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + 497, /* (600) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + 498, /* (601) query_simple_or_subquery ::= query_simple */ + 498, /* (602) query_simple_or_subquery ::= subquery */ + 423, /* (603) query_or_subquery ::= query_expression */ + 423, /* (604) query_or_subquery ::= subquery */ + 494, /* (605) order_by_clause_opt ::= */ + 494, /* (606) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 495, /* (607) slimit_clause_opt ::= */ + 495, /* (608) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + 495, /* (609) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + 495, /* (610) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 496, /* (611) limit_clause_opt ::= */ + 496, /* (612) limit_clause_opt ::= LIMIT NK_INTEGER */ + 496, /* (613) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + 496, /* (614) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 471, /* (615) subquery ::= NK_LP query_expression NK_RP */ + 471, /* (616) subquery ::= NK_LP subquery NK_RP */ + 362, /* (617) search_condition ::= common_expression */ + 499, /* (618) sort_specification_list ::= sort_specification */ + 499, /* (619) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + 500, /* (620) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 501, /* (621) ordering_specification_opt ::= */ + 501, /* (622) ordering_specification_opt ::= ASC */ + 501, /* (623) ordering_specification_opt ::= DESC */ + 502, /* (624) null_ordering_opt ::= */ + 502, /* (625) null_ordering_opt ::= NULLS FIRST */ + 502, /* (626) null_ordering_opt ::= NULLS LAST */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -4907,65 +4915,68 @@ static const signed char yyRuleInfoNRhs[] = { -2, /* (562) partition_item ::= expr_or_subquery column_alias */ -3, /* (563) partition_item ::= expr_or_subquery AS column_alias */ 0, /* (564) twindow_clause_opt ::= */ - -6, /* (565) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + -6, /* (565) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA interval_sliding_duration_literal NK_RP */ -4, /* (566) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - -6, /* (567) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - -8, /* (568) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + -6, /* (567) twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ + -8, /* (568) twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_COMMA interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ -7, /* (569) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ 0, /* (570) sliding_opt ::= */ - -4, /* (571) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - 0, /* (572) fill_opt ::= */ - -4, /* (573) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - -6, /* (574) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ - -6, /* (575) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ - -1, /* (576) fill_mode ::= NONE */ - -1, /* (577) fill_mode ::= PREV */ - -1, /* (578) fill_mode ::= NULL */ - -1, /* (579) fill_mode ::= NULL_F */ - -1, /* (580) fill_mode ::= LINEAR */ - -1, /* (581) fill_mode ::= NEXT */ - 0, /* (582) group_by_clause_opt ::= */ - -3, /* (583) group_by_clause_opt ::= GROUP BY group_by_list */ - -1, /* (584) group_by_list ::= expr_or_subquery */ - -3, /* (585) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - 0, /* (586) having_clause_opt ::= */ - -2, /* (587) having_clause_opt ::= HAVING search_condition */ - 0, /* (588) range_opt ::= */ - -6, /* (589) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - -4, /* (590) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ - 0, /* (591) every_opt ::= */ - -4, /* (592) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - -4, /* (593) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - -1, /* (594) query_simple ::= query_specification */ - -1, /* (595) query_simple ::= union_query_expression */ - -4, /* (596) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - -3, /* (597) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - -1, /* (598) query_simple_or_subquery ::= query_simple */ - -1, /* (599) query_simple_or_subquery ::= subquery */ - -1, /* (600) query_or_subquery ::= query_expression */ - -1, /* (601) query_or_subquery ::= subquery */ - 0, /* (602) order_by_clause_opt ::= */ - -3, /* (603) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 0, /* (604) slimit_clause_opt ::= */ - -2, /* (605) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - -4, /* (606) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - -4, /* (607) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 0, /* (608) limit_clause_opt ::= */ - -2, /* (609) limit_clause_opt ::= LIMIT NK_INTEGER */ - -4, /* (610) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - -4, /* (611) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - -3, /* (612) subquery ::= NK_LP query_expression NK_RP */ - -3, /* (613) subquery ::= NK_LP subquery NK_RP */ - -1, /* (614) search_condition ::= common_expression */ - -1, /* (615) sort_specification_list ::= sort_specification */ - -3, /* (616) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - -3, /* (617) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 0, /* (618) ordering_specification_opt ::= */ - -1, /* (619) ordering_specification_opt ::= ASC */ - -1, /* (620) ordering_specification_opt ::= DESC */ - 0, /* (621) null_ordering_opt ::= */ - -2, /* (622) null_ordering_opt ::= NULLS FIRST */ - -2, /* (623) null_ordering_opt ::= NULLS LAST */ + -4, /* (571) sliding_opt ::= SLIDING NK_LP interval_sliding_duration_literal NK_RP */ + -1, /* (572) interval_sliding_duration_literal ::= NK_VARIABLE */ + -1, /* (573) interval_sliding_duration_literal ::= NK_STRING */ + -1, /* (574) interval_sliding_duration_literal ::= NK_INTEGER */ + 0, /* (575) fill_opt ::= */ + -4, /* (576) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + -6, /* (577) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + -6, /* (578) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + -1, /* (579) fill_mode ::= NONE */ + -1, /* (580) fill_mode ::= PREV */ + -1, /* (581) fill_mode ::= NULL */ + -1, /* (582) fill_mode ::= NULL_F */ + -1, /* (583) fill_mode ::= LINEAR */ + -1, /* (584) fill_mode ::= NEXT */ + 0, /* (585) group_by_clause_opt ::= */ + -3, /* (586) group_by_clause_opt ::= GROUP BY group_by_list */ + -1, /* (587) group_by_list ::= expr_or_subquery */ + -3, /* (588) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + 0, /* (589) having_clause_opt ::= */ + -2, /* (590) having_clause_opt ::= HAVING search_condition */ + 0, /* (591) range_opt ::= */ + -6, /* (592) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + -4, /* (593) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 0, /* (594) every_opt ::= */ + -4, /* (595) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + -4, /* (596) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + -1, /* (597) query_simple ::= query_specification */ + -1, /* (598) query_simple ::= union_query_expression */ + -4, /* (599) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + -3, /* (600) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + -1, /* (601) query_simple_or_subquery ::= query_simple */ + -1, /* (602) query_simple_or_subquery ::= subquery */ + -1, /* (603) query_or_subquery ::= query_expression */ + -1, /* (604) query_or_subquery ::= subquery */ + 0, /* (605) order_by_clause_opt ::= */ + -3, /* (606) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 0, /* (607) slimit_clause_opt ::= */ + -2, /* (608) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + -4, /* (609) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + -4, /* (610) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 0, /* (611) limit_clause_opt ::= */ + -2, /* (612) limit_clause_opt ::= LIMIT NK_INTEGER */ + -4, /* (613) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + -4, /* (614) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + -3, /* (615) subquery ::= NK_LP query_expression NK_RP */ + -3, /* (616) subquery ::= NK_LP subquery NK_RP */ + -1, /* (617) search_condition ::= common_expression */ + -1, /* (618) sort_specification_list ::= sort_specification */ + -3, /* (619) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + -3, /* (620) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 0, /* (621) ordering_specification_opt ::= */ + -1, /* (622) ordering_specification_opt ::= ASC */ + -1, /* (623) ordering_specification_opt ::= DESC */ + 0, /* (624) null_ordering_opt ::= */ + -2, /* (625) null_ordering_opt ::= NULLS FIRST */ + -2, /* (626) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -4995,6 +5006,55 @@ static YYACTIONTYPE yy_reduce( (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); +#ifndef NDEBUG + if( yyTraceFILE ){ + yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } + yymsp = yypParser->yytos; + } +#endif + } switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -5010,19 +5070,19 @@ static YYACTIONTYPE yy_reduce( case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ #line 50 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -#line 5013 "sql.c" +#line 5073 "sql.c" yy_destructor(yypParser,345,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ #line 51 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -#line 5019 "sql.c" +#line 5079 "sql.c" yy_destructor(yypParser,346,&yymsp[0].minor); break; case 2: /* account_options ::= */ #line 55 "sql.y" { } -#line 5025 "sql.c" +#line 5085 "sql.c" break; case 3: /* account_options ::= account_options PPS literal */ case 4: /* account_options ::= account_options TSERIES literal */ yytestcase(yyruleno==4); @@ -5036,7 +5096,7 @@ static YYACTIONTYPE yy_reduce( { yy_destructor(yypParser,345,&yymsp[-2].minor); #line 56 "sql.y" { } -#line 5039 "sql.c" +#line 5099 "sql.c" yy_destructor(yypParser,347,&yymsp[0].minor); } break; @@ -5044,14 +5104,14 @@ static YYACTIONTYPE yy_reduce( { yy_destructor(yypParser,348,&yymsp[0].minor); #line 68 "sql.y" { } -#line 5047 "sql.c" +#line 5107 "sql.c" } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ { yy_destructor(yypParser,346,&yymsp[-1].minor); #line 69 "sql.y" { } -#line 5054 "sql.c" +#line 5114 "sql.c" yy_destructor(yypParser,348,&yymsp[0].minor); } break; @@ -5067,25 +5127,25 @@ static YYACTIONTYPE yy_reduce( case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); #line 73 "sql.y" { } -#line 5070 "sql.c" +#line 5130 "sql.c" yy_destructor(yypParser,347,&yymsp[0].minor); break; case 24: /* ip_range_list ::= NK_STRING */ #line 86 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } -#line 5076 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } +#line 5136 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 25: /* ip_range_list ::= ip_range_list NK_COMMA NK_STRING */ #line 87 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-2].minor.yy860, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } -#line 5082 "sql.c" - yymsp[-2].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-2].minor.yy20, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } +#line 5142 "sql.c" + yymsp[-2].minor.yy20 = yylhsminor.yy20; break; case 26: /* white_list ::= HOST ip_range_list */ #line 91 "sql.y" -{ yymsp[-1].minor.yy860 = yymsp[0].minor.yy860; } -#line 5088 "sql.c" +{ yymsp[-1].minor.yy20 = yymsp[0].minor.yy20; } +#line 5148 "sql.c" break; case 27: /* white_list_opt ::= */ case 184: /* specific_cols_opt ::= */ yytestcase(yyruleno==184); @@ -5094,135 +5154,135 @@ static YYACTIONTYPE yy_reduce( case 358: /* col_list_opt ::= */ yytestcase(yyruleno==358); case 360: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==360); case 557: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==557); - case 582: /* group_by_clause_opt ::= */ yytestcase(yyruleno==582); - case 602: /* order_by_clause_opt ::= */ yytestcase(yyruleno==602); + case 585: /* group_by_clause_opt ::= */ yytestcase(yyruleno==585); + case 605: /* order_by_clause_opt ::= */ yytestcase(yyruleno==605); #line 95 "sql.y" -{ yymsp[1].minor.yy860 = NULL; } -#line 5101 "sql.c" +{ yymsp[1].minor.yy20 = NULL; } +#line 5161 "sql.c" break; case 28: /* white_list_opt ::= white_list */ case 216: /* tags_def_opt ::= tags_def */ yytestcase(yyruleno==216); case 361: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==361); case 482: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==482); #line 96 "sql.y" -{ yylhsminor.yy860 = yymsp[0].minor.yy860; } -#line 5109 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = yymsp[0].minor.yy20; } +#line 5169 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 29: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt white_list_opt */ #line 100 "sql.y" { - pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-4].minor.yy269, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy371); - pCxt->pRootNode = addCreateUserStmtWhiteList(pCxt, pCxt->pRootNode, yymsp[0].minor.yy860); + pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-4].minor.yy479, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy541); + pCxt->pRootNode = addCreateUserStmtWhiteList(pCxt, pCxt->pRootNode, yymsp[0].minor.yy20); } -#line 5118 "sql.c" +#line 5178 "sql.c" break; case 30: /* cmd ::= ALTER USER user_name PASS NK_STRING */ #line 104 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy269, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } -#line 5123 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy479, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +#line 5183 "sql.c" break; case 31: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ #line 105 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy269, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } -#line 5128 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy479, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +#line 5188 "sql.c" break; case 32: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ #line 106 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy269, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } -#line 5133 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy479, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +#line 5193 "sql.c" break; case 33: /* cmd ::= ALTER USER user_name ADD white_list */ #line 107 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy269, TSDB_ALTER_USER_ADD_WHITE_LIST, yymsp[0].minor.yy860); } -#line 5138 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy479, TSDB_ALTER_USER_ADD_WHITE_LIST, yymsp[0].minor.yy20); } +#line 5198 "sql.c" break; case 34: /* cmd ::= ALTER USER user_name DROP white_list */ #line 108 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy269, TSDB_ALTER_USER_DROP_WHITE_LIST, yymsp[0].minor.yy860); } -#line 5143 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy479, TSDB_ALTER_USER_DROP_WHITE_LIST, yymsp[0].minor.yy20); } +#line 5203 "sql.c" break; case 35: /* cmd ::= DROP USER user_name */ #line 109 "sql.y" -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy269); } -#line 5148 "sql.c" +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy479); } +#line 5208 "sql.c" break; case 36: /* sysinfo_opt ::= */ #line 113 "sql.y" -{ yymsp[1].minor.yy371 = 1; } -#line 5153 "sql.c" +{ yymsp[1].minor.yy541 = 1; } +#line 5213 "sql.c" break; case 37: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ #line 114 "sql.y" -{ yymsp[-1].minor.yy371 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } -#line 5158 "sql.c" +{ yymsp[-1].minor.yy541 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +#line 5218 "sql.c" break; case 38: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ #line 117 "sql.y" -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy537, &yymsp[-3].minor.yy993, &yymsp[0].minor.yy269, yymsp[-2].minor.yy348); } -#line 5163 "sql.c" +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy669, &yymsp[-3].minor.yy157, &yymsp[0].minor.yy479, yymsp[-2].minor.yy342); } +#line 5223 "sql.c" break; case 39: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ #line 118 "sql.y" -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy537, &yymsp[-3].minor.yy993, &yymsp[0].minor.yy269, yymsp[-2].minor.yy348); } -#line 5168 "sql.c" +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy669, &yymsp[-3].minor.yy157, &yymsp[0].minor.yy479, yymsp[-2].minor.yy342); } +#line 5228 "sql.c" break; case 40: /* privileges ::= ALL */ #line 122 "sql.y" -{ yymsp[0].minor.yy537 = PRIVILEGE_TYPE_ALL; } -#line 5173 "sql.c" +{ yymsp[0].minor.yy669 = PRIVILEGE_TYPE_ALL; } +#line 5233 "sql.c" break; case 41: /* privileges ::= priv_type_list */ case 43: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==43); #line 123 "sql.y" -{ yylhsminor.yy537 = yymsp[0].minor.yy537; } -#line 5179 "sql.c" - yymsp[0].minor.yy537 = yylhsminor.yy537; +{ yylhsminor.yy669 = yymsp[0].minor.yy669; } +#line 5239 "sql.c" + yymsp[0].minor.yy669 = yylhsminor.yy669; break; case 42: /* privileges ::= SUBSCRIBE */ #line 124 "sql.y" -{ yymsp[0].minor.yy537 = PRIVILEGE_TYPE_SUBSCRIBE; } -#line 5185 "sql.c" +{ yymsp[0].minor.yy669 = PRIVILEGE_TYPE_SUBSCRIBE; } +#line 5245 "sql.c" break; case 44: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ #line 129 "sql.y" -{ yylhsminor.yy537 = yymsp[-2].minor.yy537 | yymsp[0].minor.yy537; } -#line 5190 "sql.c" - yymsp[-2].minor.yy537 = yylhsminor.yy537; +{ yylhsminor.yy669 = yymsp[-2].minor.yy669 | yymsp[0].minor.yy669; } +#line 5250 "sql.c" + yymsp[-2].minor.yy669 = yylhsminor.yy669; break; case 45: /* priv_type ::= READ */ #line 133 "sql.y" -{ yymsp[0].minor.yy537 = PRIVILEGE_TYPE_READ; } -#line 5196 "sql.c" +{ yymsp[0].minor.yy669 = PRIVILEGE_TYPE_READ; } +#line 5256 "sql.c" break; case 46: /* priv_type ::= WRITE */ #line 134 "sql.y" -{ yymsp[0].minor.yy537 = PRIVILEGE_TYPE_WRITE; } -#line 5201 "sql.c" +{ yymsp[0].minor.yy669 = PRIVILEGE_TYPE_WRITE; } +#line 5261 "sql.c" break; case 47: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ #line 138 "sql.y" -{ yylhsminor.yy993.first = yymsp[-2].minor.yy0; yylhsminor.yy993.second = yymsp[0].minor.yy0; } -#line 5206 "sql.c" - yymsp[-2].minor.yy993 = yylhsminor.yy993; +{ yylhsminor.yy157.first = yymsp[-2].minor.yy0; yylhsminor.yy157.second = yymsp[0].minor.yy0; } +#line 5266 "sql.c" + yymsp[-2].minor.yy157 = yylhsminor.yy157; break; case 48: /* priv_level ::= db_name NK_DOT NK_STAR */ #line 139 "sql.y" -{ yylhsminor.yy993.first = yymsp[-2].minor.yy269; yylhsminor.yy993.second = yymsp[0].minor.yy0; } -#line 5212 "sql.c" - yymsp[-2].minor.yy993 = yylhsminor.yy993; +{ yylhsminor.yy157.first = yymsp[-2].minor.yy479; yylhsminor.yy157.second = yymsp[0].minor.yy0; } +#line 5272 "sql.c" + yymsp[-2].minor.yy157 = yylhsminor.yy157; break; case 49: /* priv_level ::= db_name NK_DOT table_name */ #line 140 "sql.y" -{ yylhsminor.yy993.first = yymsp[-2].minor.yy269; yylhsminor.yy993.second = yymsp[0].minor.yy269; } -#line 5218 "sql.c" - yymsp[-2].minor.yy993 = yylhsminor.yy993; +{ yylhsminor.yy157.first = yymsp[-2].minor.yy479; yylhsminor.yy157.second = yymsp[0].minor.yy479; } +#line 5278 "sql.c" + yymsp[-2].minor.yy157 = yylhsminor.yy157; break; case 50: /* priv_level ::= topic_name */ #line 141 "sql.y" -{ yylhsminor.yy993.first = yymsp[0].minor.yy269; yylhsminor.yy993.second = nil_token; } -#line 5224 "sql.c" - yymsp[0].minor.yy993 = yylhsminor.yy993; +{ yylhsminor.yy157.first = yymsp[0].minor.yy479; yylhsminor.yy157.second = nil_token; } +#line 5284 "sql.c" + yymsp[0].minor.yy157 = yylhsminor.yy157; break; case 51: /* with_opt ::= */ case 153: /* start_opt ::= */ yytestcase(yyruleno==153); @@ -5234,78 +5294,78 @@ static YYACTIONTYPE yy_reduce( case 555: /* where_clause_opt ::= */ yytestcase(yyruleno==555); case 564: /* twindow_clause_opt ::= */ yytestcase(yyruleno==564); case 570: /* sliding_opt ::= */ yytestcase(yyruleno==570); - case 572: /* fill_opt ::= */ yytestcase(yyruleno==572); - case 586: /* having_clause_opt ::= */ yytestcase(yyruleno==586); - case 588: /* range_opt ::= */ yytestcase(yyruleno==588); - case 591: /* every_opt ::= */ yytestcase(yyruleno==591); - case 604: /* slimit_clause_opt ::= */ yytestcase(yyruleno==604); - case 608: /* limit_clause_opt ::= */ yytestcase(yyruleno==608); + case 575: /* fill_opt ::= */ yytestcase(yyruleno==575); + case 589: /* having_clause_opt ::= */ yytestcase(yyruleno==589); + case 591: /* range_opt ::= */ yytestcase(yyruleno==591); + case 594: /* every_opt ::= */ yytestcase(yyruleno==594); + case 607: /* slimit_clause_opt ::= */ yytestcase(yyruleno==607); + case 611: /* limit_clause_opt ::= */ yytestcase(yyruleno==611); #line 143 "sql.y" -{ yymsp[1].minor.yy348 = NULL; } -#line 5245 "sql.c" +{ yymsp[1].minor.yy342 = NULL; } +#line 5305 "sql.c" break; case 52: /* with_opt ::= WITH search_condition */ case 523: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==523); case 556: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==556); - case 587: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==587); + case 590: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==590); #line 144 "sql.y" -{ yymsp[-1].minor.yy348 = yymsp[0].minor.yy348; } -#line 5253 "sql.c" +{ yymsp[-1].minor.yy342 = yymsp[0].minor.yy342; } +#line 5313 "sql.c" break; case 53: /* cmd ::= CREATE DNODE dnode_endpoint */ #line 147 "sql.y" -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy269, NULL); } -#line 5258 "sql.c" +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy479, NULL); } +#line 5318 "sql.c" break; case 54: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ #line 148 "sql.y" -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy0); } -#line 5263 "sql.c" +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy0); } +#line 5323 "sql.c" break; case 55: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ #line 149 "sql.y" -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy345, false); } -#line 5268 "sql.c" +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy829, false); } +#line 5328 "sql.c" break; case 56: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ #line 150 "sql.y" -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy269, yymsp[0].minor.yy345, false); } -#line 5273 "sql.c" +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy479, yymsp[0].minor.yy829, false); } +#line 5333 "sql.c" break; case 57: /* cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ #line 151 "sql.y" -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy345); } -#line 5278 "sql.c" +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy829); } +#line 5338 "sql.c" break; case 58: /* cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ #line 152 "sql.y" -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy269, false, yymsp[0].minor.yy345); } -#line 5283 "sql.c" +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy479, false, yymsp[0].minor.yy829); } +#line 5343 "sql.c" break; case 59: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ #line 153 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } -#line 5288 "sql.c" +#line 5348 "sql.c" break; case 60: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ #line 154 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 5293 "sql.c" +#line 5353 "sql.c" break; case 61: /* cmd ::= ALTER ALL DNODES NK_STRING */ #line 155 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } -#line 5298 "sql.c" +#line 5358 "sql.c" break; case 62: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ #line 156 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 5303 "sql.c" +#line 5363 "sql.c" break; case 63: /* cmd ::= RESTORE DNODE NK_INTEGER */ #line 157 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_DNODE_STMT, &yymsp[0].minor.yy0); } -#line 5308 "sql.c" +#line 5368 "sql.c" break; case 64: /* dnode_endpoint ::= NK_STRING */ case 65: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==65); @@ -5339,9 +5399,9 @@ static YYACTIONTYPE yy_reduce( case 479: /* star_func ::= LAST */ yytestcase(yyruleno==479); case 480: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==480); #line 161 "sql.y" -{ yylhsminor.yy269 = yymsp[0].minor.yy0; } -#line 5343 "sql.c" - yymsp[0].minor.yy269 = yylhsminor.yy269; +{ yylhsminor.yy479 = yymsp[0].minor.yy0; } +#line 5403 "sql.c" + yymsp[0].minor.yy479 = yylhsminor.yy479; break; case 67: /* force_opt ::= */ case 91: /* not_exists_opt ::= */ yytestcase(yyruleno==91); @@ -5353,8 +5413,8 @@ static YYACTIONTYPE yy_reduce( case 543: /* tag_mode_opt ::= */ yytestcase(yyruleno==543); case 545: /* set_quantifier_opt ::= */ yytestcase(yyruleno==545); #line 167 "sql.y" -{ yymsp[1].minor.yy345 = false; } -#line 5357 "sql.c" +{ yymsp[1].minor.yy829 = false; } +#line 5417 "sql.c" break; case 68: /* force_opt ::= FORCE */ case 69: /* unsafe_opt ::= UNSAFE */ yytestcase(yyruleno==69); @@ -5363,430 +5423,430 @@ static YYACTIONTYPE yy_reduce( case 544: /* tag_mode_opt ::= TAGS */ yytestcase(yyruleno==544); case 546: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==546); #line 168 "sql.y" -{ yymsp[0].minor.yy345 = true; } -#line 5367 "sql.c" +{ yymsp[0].minor.yy829 = true; } +#line 5427 "sql.c" break; case 70: /* cmd ::= ALTER LOCAL NK_STRING */ #line 175 "sql.y" { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 5372 "sql.c" +#line 5432 "sql.c" break; case 71: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ #line 176 "sql.y" { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 5377 "sql.c" +#line 5437 "sql.c" break; case 72: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ #line 179 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); } -#line 5382 "sql.c" +#line 5442 "sql.c" break; case 73: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ #line 180 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); } -#line 5387 "sql.c" +#line 5447 "sql.c" break; case 74: /* cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ #line 181 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_QNODE_STMT, &yymsp[0].minor.yy0); } -#line 5392 "sql.c" +#line 5452 "sql.c" break; case 75: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ #line 184 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); } -#line 5397 "sql.c" +#line 5457 "sql.c" break; case 76: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ #line 185 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); } -#line 5402 "sql.c" +#line 5462 "sql.c" break; case 77: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ #line 188 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); } -#line 5407 "sql.c" +#line 5467 "sql.c" break; case 78: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ #line 189 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); } -#line 5412 "sql.c" +#line 5472 "sql.c" break; case 79: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ #line 192 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); } -#line 5417 "sql.c" +#line 5477 "sql.c" break; case 80: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ #line 193 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } -#line 5422 "sql.c" +#line 5482 "sql.c" break; case 81: /* cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ #line 194 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_MNODE_STMT, &yymsp[0].minor.yy0); } -#line 5427 "sql.c" +#line 5487 "sql.c" break; case 82: /* cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ #line 197 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_VNODE_STMT, &yymsp[0].minor.yy0); } -#line 5432 "sql.c" +#line 5492 "sql.c" break; case 83: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ #line 200 "sql.y" -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy345, &yymsp[-1].minor.yy269, yymsp[0].minor.yy348); } -#line 5437 "sql.c" +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy829, &yymsp[-1].minor.yy479, yymsp[0].minor.yy342); } +#line 5497 "sql.c" break; case 84: /* cmd ::= DROP DATABASE exists_opt db_name */ #line 201 "sql.y" -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy269); } -#line 5442 "sql.c" +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy829, &yymsp[0].minor.yy479); } +#line 5502 "sql.c" break; case 85: /* cmd ::= USE db_name */ #line 202 "sql.y" -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy269); } -#line 5447 "sql.c" +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy479); } +#line 5507 "sql.c" break; case 86: /* cmd ::= ALTER DATABASE db_name alter_db_options */ #line 203 "sql.y" -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy269, yymsp[0].minor.yy348); } -#line 5452 "sql.c" +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy479, yymsp[0].minor.yy342); } +#line 5512 "sql.c" break; case 87: /* cmd ::= FLUSH DATABASE db_name */ #line 204 "sql.y" -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy269); } -#line 5457 "sql.c" +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy479); } +#line 5517 "sql.c" break; case 88: /* cmd ::= TRIM DATABASE db_name speed_opt */ #line 205 "sql.y" -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy269, yymsp[0].minor.yy88); } -#line 5462 "sql.c" +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy479, yymsp[0].minor.yy114); } +#line 5522 "sql.c" break; case 89: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ #line 206 "sql.y" -{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy269, yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 5467 "sql.c" +{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy479, yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 5527 "sql.c" break; case 90: /* not_exists_opt ::= IF NOT EXISTS */ #line 210 "sql.y" -{ yymsp[-2].minor.yy345 = true; } -#line 5472 "sql.c" +{ yymsp[-2].minor.yy829 = true; } +#line 5532 "sql.c" break; case 92: /* exists_opt ::= IF EXISTS */ case 353: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==353); case 375: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==375); #line 215 "sql.y" -{ yymsp[-1].minor.yy345 = true; } -#line 5479 "sql.c" +{ yymsp[-1].minor.yy829 = true; } +#line 5539 "sql.c" break; case 94: /* db_options ::= */ #line 218 "sql.y" -{ yymsp[1].minor.yy348 = createDefaultDatabaseOptions(pCxt); } -#line 5484 "sql.c" +{ yymsp[1].minor.yy342 = createDefaultDatabaseOptions(pCxt); } +#line 5544 "sql.c" break; case 95: /* db_options ::= db_options BUFFER NK_INTEGER */ #line 219 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } -#line 5489 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } +#line 5549 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 96: /* db_options ::= db_options CACHEMODEL NK_STRING */ #line 220 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } -#line 5495 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } +#line 5555 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 97: /* db_options ::= db_options CACHESIZE NK_INTEGER */ #line 221 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } -#line 5501 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } +#line 5561 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 98: /* db_options ::= db_options COMP NK_INTEGER */ #line 222 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_COMP, &yymsp[0].minor.yy0); } -#line 5507 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_COMP, &yymsp[0].minor.yy0); } +#line 5567 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 99: /* db_options ::= db_options DURATION NK_INTEGER */ case 100: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==100); #line 223 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } -#line 5514 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } +#line 5574 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 101: /* db_options ::= db_options MAXROWS NK_INTEGER */ #line 225 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } -#line 5520 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } +#line 5580 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 102: /* db_options ::= db_options MINROWS NK_INTEGER */ #line 226 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } -#line 5526 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } +#line 5586 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 103: /* db_options ::= db_options KEEP integer_list */ case 104: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==104); #line 227 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_KEEP, yymsp[0].minor.yy860); } -#line 5533 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_KEEP, yymsp[0].minor.yy20); } +#line 5593 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 105: /* db_options ::= db_options PAGES NK_INTEGER */ #line 229 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } -#line 5539 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } +#line 5599 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 106: /* db_options ::= db_options PAGESIZE NK_INTEGER */ #line 230 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } -#line 5545 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } +#line 5605 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 107: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ #line 231 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } -#line 5551 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } +#line 5611 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 108: /* db_options ::= db_options PRECISION NK_STRING */ #line 232 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } -#line 5557 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } +#line 5617 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 109: /* db_options ::= db_options REPLICA NK_INTEGER */ #line 233 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } -#line 5563 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } +#line 5623 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 110: /* db_options ::= db_options VGROUPS NK_INTEGER */ #line 235 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } -#line 5569 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } +#line 5629 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 111: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ #line 236 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } -#line 5575 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } +#line 5635 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 112: /* db_options ::= db_options RETENTIONS retention_list */ #line 237 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_RETENTIONS, yymsp[0].minor.yy860); } -#line 5581 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_RETENTIONS, yymsp[0].minor.yy20); } +#line 5641 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 113: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ #line 238 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } -#line 5587 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } +#line 5647 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 114: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ #line 239 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_WAL, &yymsp[0].minor.yy0); } -#line 5593 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_WAL, &yymsp[0].minor.yy0); } +#line 5653 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 115: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ #line 240 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } -#line 5599 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } +#line 5659 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 116: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ #line 241 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } -#line 5605 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } +#line 5665 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 117: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ #line 242 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-3].minor.yy348, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-3].minor.yy342, DB_OPTION_WAL_RETENTION_PERIOD, &t); } -#line 5615 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +#line 5675 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 118: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ #line 247 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } -#line 5621 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } +#line 5681 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 119: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ #line 248 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-3].minor.yy348, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-3].minor.yy342, DB_OPTION_WAL_RETENTION_SIZE, &t); } -#line 5631 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +#line 5691 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 120: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ #line 253 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } -#line 5637 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } +#line 5697 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 121: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ #line 254 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } -#line 5643 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } +#line 5703 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 122: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ #line 255 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } -#line 5649 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } +#line 5709 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 123: /* db_options ::= db_options TABLE_PREFIX signed */ #line 256 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy348); } -#line 5655 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy342); } +#line 5715 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 124: /* db_options ::= db_options TABLE_SUFFIX signed */ #line 257 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy348); } -#line 5661 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy342); } +#line 5721 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 125: /* db_options ::= db_options KEEP_TIME_OFFSET NK_INTEGER */ #line 258 "sql.y" -{ yylhsminor.yy348 = setDatabaseOption(pCxt, yymsp[-2].minor.yy348, DB_OPTION_KEEP_TIME_OFFSET, &yymsp[0].minor.yy0); } -#line 5667 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setDatabaseOption(pCxt, yymsp[-2].minor.yy342, DB_OPTION_KEEP_TIME_OFFSET, &yymsp[0].minor.yy0); } +#line 5727 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 126: /* alter_db_options ::= alter_db_option */ #line 260 "sql.y" -{ yylhsminor.yy348 = createAlterDatabaseOptions(pCxt); yylhsminor.yy348 = setAlterDatabaseOption(pCxt, yylhsminor.yy348, &yymsp[0].minor.yy233); } -#line 5673 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterDatabaseOptions(pCxt); yylhsminor.yy342 = setAlterDatabaseOption(pCxt, yylhsminor.yy342, &yymsp[0].minor.yy857); } +#line 5733 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 127: /* alter_db_options ::= alter_db_options alter_db_option */ #line 261 "sql.y" -{ yylhsminor.yy348 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy348, &yymsp[0].minor.yy233); } -#line 5679 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy342, &yymsp[0].minor.yy857); } +#line 5739 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 128: /* alter_db_option ::= BUFFER NK_INTEGER */ #line 265 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5685 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5745 "sql.c" break; case 129: /* alter_db_option ::= CACHEMODEL NK_STRING */ #line 266 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5690 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5750 "sql.c" break; case 130: /* alter_db_option ::= CACHESIZE NK_INTEGER */ #line 267 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5695 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5755 "sql.c" break; case 131: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ #line 268 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5700 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5760 "sql.c" break; case 132: /* alter_db_option ::= KEEP integer_list */ case 133: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==133); #line 269 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_KEEP; yymsp[-1].minor.yy233.pList = yymsp[0].minor.yy860; } -#line 5706 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_KEEP; yymsp[-1].minor.yy857.pList = yymsp[0].minor.yy20; } +#line 5766 "sql.c" break; case 134: /* alter_db_option ::= PAGES NK_INTEGER */ #line 271 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_PAGES; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5711 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_PAGES; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5771 "sql.c" break; case 135: /* alter_db_option ::= REPLICA NK_INTEGER */ #line 272 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5716 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5776 "sql.c" break; case 136: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ #line 274 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_WAL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5721 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_WAL; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5781 "sql.c" break; case 137: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ #line 275 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5726 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5786 "sql.c" break; case 138: /* alter_db_option ::= MINROWS NK_INTEGER */ #line 276 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5731 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5791 "sql.c" break; case 139: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ #line 277 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5736 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5796 "sql.c" break; case 140: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ #line 278 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy233.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy233.val = t; + yymsp[-2].minor.yy857.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy857.val = t; } -#line 5745 "sql.c" +#line 5805 "sql.c" break; case 141: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ #line 283 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5750 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5810 "sql.c" break; case 142: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ #line 284 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy233.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy233.val = t; + yymsp[-2].minor.yy857.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy857.val = t; } -#line 5759 "sql.c" +#line 5819 "sql.c" break; case 143: /* alter_db_option ::= KEEP_TIME_OFFSET NK_INTEGER */ #line 289 "sql.y" -{ yymsp[-1].minor.yy233.type = DB_OPTION_KEEP_TIME_OFFSET; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 5764 "sql.c" +{ yymsp[-1].minor.yy857.type = DB_OPTION_KEEP_TIME_OFFSET; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 5824 "sql.c" break; case 144: /* integer_list ::= NK_INTEGER */ #line 293 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } -#line 5769 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 5829 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 145: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ case 387: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==387); #line 294 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-2].minor.yy860, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } -#line 5776 "sql.c" - yymsp[-2].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-2].minor.yy20, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 5836 "sql.c" + yymsp[-2].minor.yy20 = yylhsminor.yy20; break; case 146: /* variable_list ::= NK_VARIABLE */ #line 298 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 5782 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 5842 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 147: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ #line 299 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-2].minor.yy860, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 5788 "sql.c" - yymsp[-2].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-2].minor.yy20, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 5848 "sql.c" + yymsp[-2].minor.yy20 = yylhsminor.yy20; break; case 148: /* retention_list ::= retention */ case 178: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==178); @@ -5801,11 +5861,11 @@ static YYACTIONTYPE yy_reduce( case 489: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==489); case 548: /* select_list ::= select_item */ yytestcase(yyruleno==548); case 559: /* partition_list ::= partition_item */ yytestcase(yyruleno==559); - case 615: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==615); + case 618: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==618); #line 303 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, yymsp[0].minor.yy348); } -#line 5807 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createNodeList(pCxt, yymsp[0].minor.yy342); } +#line 5867 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 149: /* retention_list ::= retention_list NK_COMMA retention */ case 182: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==182); @@ -5818,1045 +5878,1045 @@ static YYACTIONTYPE yy_reduce( case 484: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==484); case 549: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==549); case 560: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==560); - case 616: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==616); + case 619: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==619); #line 304 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-2].minor.yy860, yymsp[0].minor.yy348); } -#line 5824 "sql.c" - yymsp[-2].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-2].minor.yy20, yymsp[0].minor.yy342); } +#line 5884 "sql.c" + yymsp[-2].minor.yy20 = yylhsminor.yy20; break; case 150: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ #line 306 "sql.y" -{ yylhsminor.yy348 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 5830 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 5890 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 151: /* speed_opt ::= */ case 348: /* bufsize_opt ::= */ yytestcase(yyruleno==348); #line 310 "sql.y" -{ yymsp[1].minor.yy88 = 0; } -#line 5837 "sql.c" +{ yymsp[1].minor.yy114 = 0; } +#line 5897 "sql.c" break; case 152: /* speed_opt ::= BWLIMIT NK_INTEGER */ case 349: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==349); #line 311 "sql.y" -{ yymsp[-1].minor.yy88 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } -#line 5843 "sql.c" +{ yymsp[-1].minor.yy114 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } +#line 5903 "sql.c" break; case 154: /* start_opt ::= START WITH NK_INTEGER */ case 158: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==158); #line 314 "sql.y" -{ yymsp[-2].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } -#line 5849 "sql.c" +{ yymsp[-2].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +#line 5909 "sql.c" break; case 155: /* start_opt ::= START WITH NK_STRING */ case 159: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==159); #line 315 "sql.y" -{ yymsp[-2].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 5855 "sql.c" +{ yymsp[-2].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +#line 5915 "sql.c" break; case 156: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ case 160: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==160); #line 316 "sql.y" -{ yymsp[-3].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 5861 "sql.c" +{ yymsp[-3].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +#line 5921 "sql.c" break; case 161: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 163: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==163); #line 325 "sql.y" -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy345, yymsp[-5].minor.yy348, yymsp[-3].minor.yy860, yymsp[-1].minor.yy860, yymsp[0].minor.yy348); } -#line 5867 "sql.c" +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy829, yymsp[-5].minor.yy342, yymsp[-3].minor.yy20, yymsp[-1].minor.yy20, yymsp[0].minor.yy342); } +#line 5927 "sql.c" break; case 162: /* cmd ::= CREATE TABLE multi_create_clause */ #line 326 "sql.y" -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy860); } -#line 5872 "sql.c" +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy20); } +#line 5932 "sql.c" break; case 164: /* cmd ::= DROP TABLE multi_drop_clause */ #line 329 "sql.y" -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy860); } -#line 5877 "sql.c" +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy20); } +#line 5937 "sql.c" break; case 165: /* cmd ::= DROP STABLE exists_opt full_table_name */ #line 330 "sql.y" -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy345, yymsp[0].minor.yy348); } -#line 5882 "sql.c" +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy829, yymsp[0].minor.yy342); } +#line 5942 "sql.c" break; case 166: /* cmd ::= ALTER TABLE alter_table_clause */ case 389: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==389); case 390: /* cmd ::= insert_query */ yytestcase(yyruleno==390); #line 332 "sql.y" -{ pCxt->pRootNode = yymsp[0].minor.yy348; } -#line 5889 "sql.c" +{ pCxt->pRootNode = yymsp[0].minor.yy342; } +#line 5949 "sql.c" break; case 167: /* cmd ::= ALTER STABLE alter_table_clause */ #line 333 "sql.y" -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy348); } -#line 5894 "sql.c" +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy342); } +#line 5954 "sql.c" break; case 168: /* alter_table_clause ::= full_table_name alter_table_options */ #line 335 "sql.y" -{ yylhsminor.yy348 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 5899 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 5959 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 169: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ #line 337 "sql.y" -{ yylhsminor.yy348 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy348, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy269, yymsp[0].minor.yy720); } -#line 5905 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy342, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy479, yymsp[0].minor.yy750); } +#line 5965 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 170: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ #line 338 "sql.y" -{ yylhsminor.yy348 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy348, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy269); } -#line 5911 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy342, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy479); } +#line 5971 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 171: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ #line 340 "sql.y" -{ yylhsminor.yy348 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy348, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy269, yymsp[0].minor.yy720); } -#line 5917 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy342, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy479, yymsp[0].minor.yy750); } +#line 5977 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 172: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ #line 342 "sql.y" -{ yylhsminor.yy348 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy348, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy269, &yymsp[0].minor.yy269); } -#line 5923 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy342, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy479, &yymsp[0].minor.yy479); } +#line 5983 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 173: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ #line 344 "sql.y" -{ yylhsminor.yy348 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy348, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy269, yymsp[0].minor.yy720); } -#line 5929 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy342, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy479, yymsp[0].minor.yy750); } +#line 5989 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 174: /* alter_table_clause ::= full_table_name DROP TAG column_name */ #line 345 "sql.y" -{ yylhsminor.yy348 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy348, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy269); } -#line 5935 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy342, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy479); } +#line 5995 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 175: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ #line 347 "sql.y" -{ yylhsminor.yy348 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy348, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy269, yymsp[0].minor.yy720); } -#line 5941 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy342, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy479, yymsp[0].minor.yy750); } +#line 6001 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 176: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ #line 349 "sql.y" -{ yylhsminor.yy348 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy348, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy269, &yymsp[0].minor.yy269); } -#line 5947 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy342, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy479, &yymsp[0].minor.yy479); } +#line 6007 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 177: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ #line 351 "sql.y" -{ yylhsminor.yy348 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy348, &yymsp[-2].minor.yy269, yymsp[0].minor.yy348); } -#line 5953 "sql.c" - yymsp[-5].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy342, &yymsp[-2].minor.yy479, yymsp[0].minor.yy342); } +#line 6013 "sql.c" + yymsp[-5].minor.yy342 = yylhsminor.yy342; break; case 179: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 490: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==490); #line 356 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-1].minor.yy860, yymsp[0].minor.yy348); } -#line 5960 "sql.c" - yymsp[-1].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-1].minor.yy20, yymsp[0].minor.yy342); } +#line 6020 "sql.c" + yymsp[-1].minor.yy20 = yylhsminor.yy20; break; case 180: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ #line 360 "sql.y" -{ yylhsminor.yy348 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy345, yymsp[-8].minor.yy348, yymsp[-6].minor.yy348, yymsp[-5].minor.yy860, yymsp[-2].minor.yy860, yymsp[0].minor.yy348); } -#line 5966 "sql.c" - yymsp[-9].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy829, yymsp[-8].minor.yy342, yymsp[-6].minor.yy342, yymsp[-5].minor.yy20, yymsp[-2].minor.yy20, yymsp[0].minor.yy342); } +#line 6026 "sql.c" + yymsp[-9].minor.yy342 = yylhsminor.yy342; break; case 183: /* drop_table_clause ::= exists_opt full_table_name */ #line 367 "sql.y" -{ yylhsminor.yy348 = createDropTableClause(pCxt, yymsp[-1].minor.yy345, yymsp[0].minor.yy348); } -#line 5972 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createDropTableClause(pCxt, yymsp[-1].minor.yy829, yymsp[0].minor.yy342); } +#line 6032 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 185: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ case 359: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==359); #line 372 "sql.y" -{ yymsp[-2].minor.yy860 = yymsp[-1].minor.yy860; } -#line 5979 "sql.c" +{ yymsp[-2].minor.yy20 = yymsp[-1].minor.yy20; } +#line 6039 "sql.c" break; case 186: /* full_table_name ::= table_name */ #line 374 "sql.y" -{ yylhsminor.yy348 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy269, NULL); } -#line 5984 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy479, NULL); } +#line 6044 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 187: /* full_table_name ::= db_name NK_DOT table_name */ #line 375 "sql.y" -{ yylhsminor.yy348 = createRealTableNode(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy269, NULL); } -#line 5990 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRealTableNode(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy479, NULL); } +#line 6050 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 190: /* column_def ::= column_name type_name */ #line 382 "sql.y" -{ yylhsminor.yy348 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy269, yymsp[0].minor.yy720, NULL); } -#line 5996 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy479, yymsp[0].minor.yy750, NULL); } +#line 6056 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 191: /* type_name ::= BOOL */ #line 387 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_BOOL); } -#line 6002 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_BOOL); } +#line 6062 "sql.c" break; case 192: /* type_name ::= TINYINT */ #line 388 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_TINYINT); } -#line 6007 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_TINYINT); } +#line 6067 "sql.c" break; case 193: /* type_name ::= SMALLINT */ #line 389 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_SMALLINT); } -#line 6012 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +#line 6072 "sql.c" break; case 194: /* type_name ::= INT */ case 195: /* type_name ::= INTEGER */ yytestcase(yyruleno==195); #line 390 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_INT); } -#line 6018 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_INT); } +#line 6078 "sql.c" break; case 196: /* type_name ::= BIGINT */ #line 392 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_BIGINT); } -#line 6023 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_BIGINT); } +#line 6083 "sql.c" break; case 197: /* type_name ::= FLOAT */ #line 393 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_FLOAT); } -#line 6028 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_FLOAT); } +#line 6088 "sql.c" break; case 198: /* type_name ::= DOUBLE */ #line 394 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_DOUBLE); } -#line 6033 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +#line 6093 "sql.c" break; case 199: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ #line 395 "sql.y" -{ yymsp[-3].minor.yy720 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } -#line 6038 "sql.c" +{ yymsp[-3].minor.yy750 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +#line 6098 "sql.c" break; case 200: /* type_name ::= TIMESTAMP */ #line 396 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } -#line 6043 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +#line 6103 "sql.c" break; case 201: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ #line 397 "sql.y" -{ yymsp[-3].minor.yy720 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } -#line 6048 "sql.c" +{ yymsp[-3].minor.yy750 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +#line 6108 "sql.c" break; case 202: /* type_name ::= TINYINT UNSIGNED */ #line 398 "sql.y" -{ yymsp[-1].minor.yy720 = createDataType(TSDB_DATA_TYPE_UTINYINT); } -#line 6053 "sql.c" +{ yymsp[-1].minor.yy750 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +#line 6113 "sql.c" break; case 203: /* type_name ::= SMALLINT UNSIGNED */ #line 399 "sql.y" -{ yymsp[-1].minor.yy720 = createDataType(TSDB_DATA_TYPE_USMALLINT); } -#line 6058 "sql.c" +{ yymsp[-1].minor.yy750 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +#line 6118 "sql.c" break; case 204: /* type_name ::= INT UNSIGNED */ #line 400 "sql.y" -{ yymsp[-1].minor.yy720 = createDataType(TSDB_DATA_TYPE_UINT); } -#line 6063 "sql.c" +{ yymsp[-1].minor.yy750 = createDataType(TSDB_DATA_TYPE_UINT); } +#line 6123 "sql.c" break; case 205: /* type_name ::= BIGINT UNSIGNED */ #line 401 "sql.y" -{ yymsp[-1].minor.yy720 = createDataType(TSDB_DATA_TYPE_UBIGINT); } -#line 6068 "sql.c" +{ yymsp[-1].minor.yy750 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +#line 6128 "sql.c" break; case 206: /* type_name ::= JSON */ #line 402 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_JSON); } -#line 6073 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_JSON); } +#line 6133 "sql.c" break; case 207: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ #line 403 "sql.y" -{ yymsp[-3].minor.yy720 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } -#line 6078 "sql.c" +{ yymsp[-3].minor.yy750 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +#line 6138 "sql.c" break; case 208: /* type_name ::= MEDIUMBLOB */ #line 404 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } -#line 6083 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +#line 6143 "sql.c" break; case 209: /* type_name ::= BLOB */ #line 405 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_BLOB); } -#line 6088 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_BLOB); } +#line 6148 "sql.c" break; case 210: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ #line 406 "sql.y" -{ yymsp[-3].minor.yy720 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } -#line 6093 "sql.c" +{ yymsp[-3].minor.yy750 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +#line 6153 "sql.c" break; case 211: /* type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ #line 407 "sql.y" -{ yymsp[-3].minor.yy720 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); } -#line 6098 "sql.c" +{ yymsp[-3].minor.yy750 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); } +#line 6158 "sql.c" break; case 212: /* type_name ::= DECIMAL */ #line 408 "sql.y" -{ yymsp[0].minor.yy720 = createDataType(TSDB_DATA_TYPE_DECIMAL); } -#line 6103 "sql.c" +{ yymsp[0].minor.yy750 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +#line 6163 "sql.c" break; case 213: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ #line 409 "sql.y" -{ yymsp[-3].minor.yy720 = createDataType(TSDB_DATA_TYPE_DECIMAL); } -#line 6108 "sql.c" +{ yymsp[-3].minor.yy750 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +#line 6168 "sql.c" break; case 214: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ #line 410 "sql.y" -{ yymsp[-5].minor.yy720 = createDataType(TSDB_DATA_TYPE_DECIMAL); } -#line 6113 "sql.c" +{ yymsp[-5].minor.yy750 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +#line 6173 "sql.c" break; case 217: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ case 362: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==362); #line 419 "sql.y" -{ yymsp[-3].minor.yy860 = yymsp[-1].minor.yy860; } -#line 6119 "sql.c" +{ yymsp[-3].minor.yy20 = yymsp[-1].minor.yy20; } +#line 6179 "sql.c" break; case 218: /* table_options ::= */ #line 421 "sql.y" -{ yymsp[1].minor.yy348 = createDefaultTableOptions(pCxt); } -#line 6124 "sql.c" +{ yymsp[1].minor.yy342 = createDefaultTableOptions(pCxt); } +#line 6184 "sql.c" break; case 219: /* table_options ::= table_options COMMENT NK_STRING */ #line 422 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-2].minor.yy348, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } -#line 6129 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-2].minor.yy342, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } +#line 6189 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 220: /* table_options ::= table_options MAX_DELAY duration_list */ #line 423 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-2].minor.yy348, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy860); } -#line 6135 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-2].minor.yy342, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy20); } +#line 6195 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 221: /* table_options ::= table_options WATERMARK duration_list */ #line 424 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-2].minor.yy348, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy860); } -#line 6141 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-2].minor.yy342, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy20); } +#line 6201 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 222: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ #line 425 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-4].minor.yy348, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy860); } -#line 6147 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-4].minor.yy342, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy20); } +#line 6207 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 223: /* table_options ::= table_options TTL NK_INTEGER */ #line 426 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-2].minor.yy348, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } -#line 6153 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-2].minor.yy342, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } +#line 6213 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 224: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ #line 427 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-4].minor.yy348, TABLE_OPTION_SMA, yymsp[-1].minor.yy860); } -#line 6159 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-4].minor.yy342, TABLE_OPTION_SMA, yymsp[-1].minor.yy20); } +#line 6219 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 225: /* table_options ::= table_options DELETE_MARK duration_list */ #line 428 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-2].minor.yy348, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy860); } -#line 6165 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-2].minor.yy342, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy20); } +#line 6225 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 226: /* alter_table_options ::= alter_table_option */ #line 430 "sql.y" -{ yylhsminor.yy348 = createAlterTableOptions(pCxt); yylhsminor.yy348 = setTableOption(pCxt, yylhsminor.yy348, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } -#line 6171 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createAlterTableOptions(pCxt); yylhsminor.yy342 = setTableOption(pCxt, yylhsminor.yy342, yymsp[0].minor.yy857.type, &yymsp[0].minor.yy857.val); } +#line 6231 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 227: /* alter_table_options ::= alter_table_options alter_table_option */ #line 431 "sql.y" -{ yylhsminor.yy348 = setTableOption(pCxt, yymsp[-1].minor.yy348, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } -#line 6177 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setTableOption(pCxt, yymsp[-1].minor.yy342, yymsp[0].minor.yy857.type, &yymsp[0].minor.yy857.val); } +#line 6237 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 228: /* alter_table_option ::= COMMENT NK_STRING */ #line 435 "sql.y" -{ yymsp[-1].minor.yy233.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 6183 "sql.c" +{ yymsp[-1].minor.yy857.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 6243 "sql.c" break; case 229: /* alter_table_option ::= TTL NK_INTEGER */ #line 436 "sql.y" -{ yymsp[-1].minor.yy233.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } -#line 6188 "sql.c" +{ yymsp[-1].minor.yy857.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy857.val = yymsp[0].minor.yy0; } +#line 6248 "sql.c" break; case 230: /* duration_list ::= duration_literal */ case 446: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==446); #line 440 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); } -#line 6194 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } +#line 6254 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 231: /* duration_list ::= duration_list NK_COMMA duration_literal */ case 447: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==447); #line 441 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-2].minor.yy860, releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); } -#line 6201 "sql.c" - yymsp[-2].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-2].minor.yy20, releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } +#line 6261 "sql.c" + yymsp[-2].minor.yy20 = yylhsminor.yy20; break; case 234: /* rollup_func_name ::= function_name */ #line 448 "sql.y" -{ yylhsminor.yy348 = createFunctionNode(pCxt, &yymsp[0].minor.yy269, NULL); } -#line 6207 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createFunctionNode(pCxt, &yymsp[0].minor.yy479, NULL); } +#line 6267 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 235: /* rollup_func_name ::= FIRST */ case 236: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==236); case 300: /* tag_item ::= QTAGS */ yytestcase(yyruleno==300); #line 449 "sql.y" -{ yylhsminor.yy348 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 6215 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } +#line 6275 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 239: /* col_name ::= column_name */ case 301: /* tag_item ::= column_name */ yytestcase(yyruleno==301); #line 457 "sql.y" -{ yylhsminor.yy348 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy269); } -#line 6222 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy479); } +#line 6282 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 240: /* cmd ::= SHOW DNODES */ #line 460 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } -#line 6228 "sql.c" +#line 6288 "sql.c" break; case 241: /* cmd ::= SHOW USERS */ #line 461 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } -#line 6233 "sql.c" +#line 6293 "sql.c" break; case 242: /* cmd ::= SHOW USER PRIVILEGES */ #line 462 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } -#line 6238 "sql.c" +#line 6298 "sql.c" break; case 243: /* cmd ::= SHOW db_kind_opt DATABASES */ #line 463 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); - setShowKind(pCxt, pCxt->pRootNode, yymsp[-1].minor.yy361); + setShowKind(pCxt, pCxt->pRootNode, yymsp[-1].minor.yy545); } -#line 6246 "sql.c" +#line 6306 "sql.c" break; case 244: /* cmd ::= SHOW table_kind_db_name_cond_opt TABLES like_pattern_opt */ #line 467 "sql.y" { - pCxt->pRootNode = createShowTablesStmt(pCxt, yymsp[-2].minor.yy749, yymsp[0].minor.yy348, OP_TYPE_LIKE); + pCxt->pRootNode = createShowTablesStmt(pCxt, yymsp[-2].minor.yy711, yymsp[0].minor.yy342, OP_TYPE_LIKE); } -#line 6253 "sql.c" +#line 6313 "sql.c" break; case 245: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ #line 470 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy348, yymsp[0].minor.yy348, OP_TYPE_LIKE); } -#line 6258 "sql.c" +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy342, yymsp[0].minor.yy342, OP_TYPE_LIKE); } +#line 6318 "sql.c" break; case 246: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ #line 471 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy348, NULL, OP_TYPE_LIKE); } -#line 6263 "sql.c" +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy342, NULL, OP_TYPE_LIKE); } +#line 6323 "sql.c" break; case 247: /* cmd ::= SHOW MNODES */ #line 472 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } -#line 6268 "sql.c" +#line 6328 "sql.c" break; case 248: /* cmd ::= SHOW QNODES */ #line 474 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } -#line 6273 "sql.c" +#line 6333 "sql.c" break; case 249: /* cmd ::= SHOW FUNCTIONS */ #line 475 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } -#line 6278 "sql.c" +#line 6338 "sql.c" break; case 250: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ #line 476 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy348, yymsp[-1].minor.yy348, OP_TYPE_EQUAL); } -#line 6283 "sql.c" +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy342, yymsp[-1].minor.yy342, OP_TYPE_EQUAL); } +#line 6343 "sql.c" break; case 251: /* cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ #line 477 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy269), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy269), OP_TYPE_EQUAL); } -#line 6288 "sql.c" +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy479), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy479), OP_TYPE_EQUAL); } +#line 6348 "sql.c" break; case 252: /* cmd ::= SHOW STREAMS */ #line 478 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } -#line 6293 "sql.c" +#line 6353 "sql.c" break; case 253: /* cmd ::= SHOW ACCOUNTS */ #line 479 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -#line 6298 "sql.c" +#line 6358 "sql.c" break; case 254: /* cmd ::= SHOW APPS */ #line 480 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } -#line 6303 "sql.c" +#line 6363 "sql.c" break; case 255: /* cmd ::= SHOW CONNECTIONS */ #line 481 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } -#line 6308 "sql.c" +#line 6368 "sql.c" break; case 256: /* cmd ::= SHOW LICENCES */ case 257: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==257); #line 482 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } -#line 6314 "sql.c" +#line 6374 "sql.c" break; case 258: /* cmd ::= SHOW CREATE DATABASE db_name */ #line 484 "sql.y" -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy269); } -#line 6319 "sql.c" +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy479); } +#line 6379 "sql.c" break; case 259: /* cmd ::= SHOW CREATE TABLE full_table_name */ #line 485 "sql.y" -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy348); } -#line 6324 "sql.c" +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy342); } +#line 6384 "sql.c" break; case 260: /* cmd ::= SHOW CREATE STABLE full_table_name */ #line 486 "sql.y" -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy348); } -#line 6329 "sql.c" +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy342); } +#line 6389 "sql.c" break; case 261: /* cmd ::= SHOW QUERIES */ #line 487 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } -#line 6334 "sql.c" +#line 6394 "sql.c" break; case 262: /* cmd ::= SHOW SCORES */ #line 488 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } -#line 6339 "sql.c" +#line 6399 "sql.c" break; case 263: /* cmd ::= SHOW TOPICS */ #line 489 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } -#line 6344 "sql.c" +#line 6404 "sql.c" break; case 264: /* cmd ::= SHOW VARIABLES */ case 265: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==265); #line 490 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } -#line 6350 "sql.c" +#line 6410 "sql.c" break; case 266: /* cmd ::= SHOW LOCAL VARIABLES */ #line 492 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } -#line 6355 "sql.c" +#line 6415 "sql.c" break; case 267: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ #line 493 "sql.y" -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy348); } -#line 6360 "sql.c" +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy342); } +#line 6420 "sql.c" break; case 268: /* cmd ::= SHOW BNODES */ #line 494 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } -#line 6365 "sql.c" +#line 6425 "sql.c" break; case 269: /* cmd ::= SHOW SNODES */ #line 495 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } -#line 6370 "sql.c" +#line 6430 "sql.c" break; case 270: /* cmd ::= SHOW CLUSTER */ #line 496 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } -#line 6375 "sql.c" +#line 6435 "sql.c" break; case 271: /* cmd ::= SHOW TRANSACTIONS */ #line 497 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } -#line 6380 "sql.c" +#line 6440 "sql.c" break; case 272: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ #line 498 "sql.y" -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy348); } -#line 6385 "sql.c" +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy342); } +#line 6445 "sql.c" break; case 273: /* cmd ::= SHOW CONSUMERS */ #line 499 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } -#line 6390 "sql.c" +#line 6450 "sql.c" break; case 274: /* cmd ::= SHOW SUBSCRIPTIONS */ #line 500 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } -#line 6395 "sql.c" +#line 6455 "sql.c" break; case 275: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ #line 501 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy348, yymsp[-1].minor.yy348, OP_TYPE_EQUAL); } -#line 6400 "sql.c" +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy342, yymsp[-1].minor.yy342, OP_TYPE_EQUAL); } +#line 6460 "sql.c" break; case 276: /* cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ #line 502 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy269), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy269), OP_TYPE_EQUAL); } -#line 6405 "sql.c" +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy479), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy479), OP_TYPE_EQUAL); } +#line 6465 "sql.c" break; case 277: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ #line 503 "sql.y" -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy348, yymsp[0].minor.yy348, yymsp[-3].minor.yy860); } -#line 6410 "sql.c" +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy342, yymsp[0].minor.yy342, yymsp[-3].minor.yy20); } +#line 6470 "sql.c" break; case 278: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ #line 504 "sql.y" -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &yymsp[0].minor.yy269), createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy269), yymsp[-4].minor.yy860); } -#line 6415 "sql.c" +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &yymsp[0].minor.yy479), createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy479), yymsp[-4].minor.yy20); } +#line 6475 "sql.c" break; case 279: /* cmd ::= SHOW VNODES ON DNODE NK_INTEGER */ #line 505 "sql.y" { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } -#line 6420 "sql.c" +#line 6480 "sql.c" break; case 280: /* cmd ::= SHOW VNODES */ #line 506 "sql.y" { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, NULL); } -#line 6425 "sql.c" +#line 6485 "sql.c" break; case 281: /* cmd ::= SHOW db_name_cond_opt ALIVE */ #line 508 "sql.y" -{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy348, QUERY_NODE_SHOW_DB_ALIVE_STMT); } -#line 6430 "sql.c" +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy342, QUERY_NODE_SHOW_DB_ALIVE_STMT); } +#line 6490 "sql.c" break; case 282: /* cmd ::= SHOW CLUSTER ALIVE */ #line 509 "sql.y" { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } -#line 6435 "sql.c" +#line 6495 "sql.c" break; case 283: /* table_kind_db_name_cond_opt ::= */ #line 513 "sql.y" -{ yymsp[1].minor.yy749.kind = SHOW_KIND_ALL; yymsp[1].minor.yy749.dbName = nil_token; } -#line 6440 "sql.c" +{ yymsp[1].minor.yy711.kind = SHOW_KIND_ALL; yymsp[1].minor.yy711.dbName = nil_token; } +#line 6500 "sql.c" break; case 284: /* table_kind_db_name_cond_opt ::= table_kind */ #line 514 "sql.y" -{ yylhsminor.yy749.kind = yymsp[0].minor.yy361; yylhsminor.yy749.dbName = nil_token; } -#line 6445 "sql.c" - yymsp[0].minor.yy749 = yylhsminor.yy749; +{ yylhsminor.yy711.kind = yymsp[0].minor.yy545; yylhsminor.yy711.dbName = nil_token; } +#line 6505 "sql.c" + yymsp[0].minor.yy711 = yylhsminor.yy711; break; case 285: /* table_kind_db_name_cond_opt ::= db_name NK_DOT */ #line 515 "sql.y" -{ yylhsminor.yy749.kind = SHOW_KIND_ALL; yylhsminor.yy749.dbName = yymsp[-1].minor.yy269; } -#line 6451 "sql.c" - yymsp[-1].minor.yy749 = yylhsminor.yy749; +{ yylhsminor.yy711.kind = SHOW_KIND_ALL; yylhsminor.yy711.dbName = yymsp[-1].minor.yy479; } +#line 6511 "sql.c" + yymsp[-1].minor.yy711 = yylhsminor.yy711; break; case 286: /* table_kind_db_name_cond_opt ::= table_kind db_name NK_DOT */ #line 516 "sql.y" -{ yylhsminor.yy749.kind = yymsp[-2].minor.yy361; yylhsminor.yy749.dbName = yymsp[-1].minor.yy269; } -#line 6457 "sql.c" - yymsp[-2].minor.yy749 = yylhsminor.yy749; +{ yylhsminor.yy711.kind = yymsp[-2].minor.yy545; yylhsminor.yy711.dbName = yymsp[-1].minor.yy479; } +#line 6517 "sql.c" + yymsp[-2].minor.yy711 = yylhsminor.yy711; break; case 287: /* table_kind ::= NORMAL */ #line 520 "sql.y" -{ yymsp[0].minor.yy361 = SHOW_KIND_TABLES_NORMAL; } -#line 6463 "sql.c" +{ yymsp[0].minor.yy545 = SHOW_KIND_TABLES_NORMAL; } +#line 6523 "sql.c" break; case 288: /* table_kind ::= CHILD */ #line 521 "sql.y" -{ yymsp[0].minor.yy361 = SHOW_KIND_TABLES_CHILD; } -#line 6468 "sql.c" +{ yymsp[0].minor.yy545 = SHOW_KIND_TABLES_CHILD; } +#line 6528 "sql.c" break; case 289: /* db_name_cond_opt ::= */ case 294: /* from_db_opt ::= */ yytestcase(yyruleno==294); #line 523 "sql.y" -{ yymsp[1].minor.yy348 = createDefaultDatabaseCondValue(pCxt); } -#line 6474 "sql.c" +{ yymsp[1].minor.yy342 = createDefaultDatabaseCondValue(pCxt); } +#line 6534 "sql.c" break; case 290: /* db_name_cond_opt ::= db_name NK_DOT */ #line 524 "sql.y" -{ yylhsminor.yy348 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy269); } -#line 6479 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy479); } +#line 6539 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 292: /* like_pattern_opt ::= LIKE NK_STRING */ #line 527 "sql.y" -{ yymsp[-1].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } -#line 6485 "sql.c" +{ yymsp[-1].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +#line 6545 "sql.c" break; case 293: /* table_name_cond ::= table_name */ #line 529 "sql.y" -{ yylhsminor.yy348 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy269); } -#line 6490 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy479); } +#line 6550 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 295: /* from_db_opt ::= FROM db_name */ #line 532 "sql.y" -{ yymsp[-1].minor.yy348 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy269); } -#line 6496 "sql.c" +{ yymsp[-1].minor.yy342 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy479); } +#line 6556 "sql.c" break; case 299: /* tag_item ::= TBNAME */ #line 540 "sql.y" -{ yylhsminor.yy348 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } -#line 6501 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } +#line 6561 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 302: /* tag_item ::= column_name column_alias */ #line 543 "sql.y" -{ yylhsminor.yy348 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy269), &yymsp[0].minor.yy269); } -#line 6507 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy479), &yymsp[0].minor.yy479); } +#line 6567 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 303: /* tag_item ::= column_name AS column_alias */ #line 544 "sql.y" -{ yylhsminor.yy348 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy269), &yymsp[0].minor.yy269); } -#line 6513 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy479), &yymsp[0].minor.yy479); } +#line 6573 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 304: /* db_kind_opt ::= */ #line 548 "sql.y" -{ yymsp[1].minor.yy361 = SHOW_KIND_ALL; } -#line 6519 "sql.c" +{ yymsp[1].minor.yy545 = SHOW_KIND_ALL; } +#line 6579 "sql.c" break; case 305: /* db_kind_opt ::= USER */ #line 549 "sql.y" -{ yymsp[0].minor.yy361 = SHOW_KIND_DATABASES_USER; } -#line 6524 "sql.c" +{ yymsp[0].minor.yy545 = SHOW_KIND_DATABASES_USER; } +#line 6584 "sql.c" break; case 306: /* db_kind_opt ::= SYSTEM */ #line 550 "sql.y" -{ yymsp[0].minor.yy361 = SHOW_KIND_DATABASES_SYSTEM; } -#line 6529 "sql.c" +{ yymsp[0].minor.yy545 = SHOW_KIND_DATABASES_SYSTEM; } +#line 6589 "sql.c" break; case 307: /* cmd ::= CREATE SMA INDEX not_exists_opt col_name ON full_table_name index_options */ #line 554 "sql.y" -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy345, yymsp[-3].minor.yy348, yymsp[-1].minor.yy348, NULL, yymsp[0].minor.yy348); } -#line 6534 "sql.c" +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy829, yymsp[-3].minor.yy342, yymsp[-1].minor.yy342, NULL, yymsp[0].minor.yy342); } +#line 6594 "sql.c" break; case 308: /* cmd ::= CREATE INDEX not_exists_opt col_name ON full_table_name NK_LP col_name_list NK_RP */ #line 556 "sql.y" -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy345, yymsp[-5].minor.yy348, yymsp[-3].minor.yy348, yymsp[-1].minor.yy860, NULL); } -#line 6539 "sql.c" +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy829, yymsp[-5].minor.yy342, yymsp[-3].minor.yy342, yymsp[-1].minor.yy20, NULL); } +#line 6599 "sql.c" break; case 309: /* cmd ::= DROP INDEX exists_opt full_index_name */ #line 557 "sql.y" -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy345, yymsp[0].minor.yy348); } -#line 6544 "sql.c" +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy829, yymsp[0].minor.yy342); } +#line 6604 "sql.c" break; case 310: /* full_index_name ::= index_name */ #line 559 "sql.y" -{ yylhsminor.yy348 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy269); } -#line 6549 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy479); } +#line 6609 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 311: /* full_index_name ::= db_name NK_DOT index_name */ #line 560 "sql.y" -{ yylhsminor.yy348 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy269); } -#line 6555 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy479); } +#line 6615 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 312: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ #line 563 "sql.y" -{ yymsp[-9].minor.yy348 = createIndexOption(pCxt, yymsp[-7].minor.yy860, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), NULL, yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 6561 "sql.c" +{ yymsp[-9].minor.yy342 = createIndexOption(pCxt, yymsp[-7].minor.yy20, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), NULL, yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 6621 "sql.c" break; case 313: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ #line 566 "sql.y" -{ yymsp[-11].minor.yy348 = createIndexOption(pCxt, yymsp[-9].minor.yy860, releaseRawExprNode(pCxt, yymsp[-5].minor.yy348), releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 6566 "sql.c" +{ yymsp[-11].minor.yy342 = createIndexOption(pCxt, yymsp[-9].minor.yy20, releaseRawExprNode(pCxt, yymsp[-5].minor.yy342), releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 6626 "sql.c" break; case 316: /* func ::= sma_func_name NK_LP expression_list NK_RP */ #line 573 "sql.y" -{ yylhsminor.yy348 = createFunctionNode(pCxt, &yymsp[-3].minor.yy269, yymsp[-1].minor.yy860); } -#line 6571 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createFunctionNode(pCxt, &yymsp[-3].minor.yy479, yymsp[-1].minor.yy20); } +#line 6631 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 317: /* sma_func_name ::= function_name */ case 533: /* alias_opt ::= table_alias */ yytestcase(yyruleno==533); #line 577 "sql.y" -{ yylhsminor.yy269 = yymsp[0].minor.yy269; } -#line 6578 "sql.c" - yymsp[0].minor.yy269 = yylhsminor.yy269; +{ yylhsminor.yy479 = yymsp[0].minor.yy479; } +#line 6638 "sql.c" + yymsp[0].minor.yy479 = yylhsminor.yy479; break; case 322: /* sma_stream_opt ::= */ case 363: /* stream_options ::= */ yytestcase(yyruleno==363); #line 583 "sql.y" -{ yymsp[1].minor.yy348 = createStreamOptions(pCxt); } -#line 6585 "sql.c" +{ yymsp[1].minor.yy342 = createStreamOptions(pCxt); } +#line 6645 "sql.c" break; case 323: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ #line 584 "sql.y" -{ ((SStreamOptions*)yymsp[-2].minor.yy348)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy348); yylhsminor.yy348 = yymsp[-2].minor.yy348; } -#line 6590 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ ((SStreamOptions*)yymsp[-2].minor.yy342)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy342); yylhsminor.yy342 = yymsp[-2].minor.yy342; } +#line 6650 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 324: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ #line 585 "sql.y" -{ ((SStreamOptions*)yymsp[-2].minor.yy348)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy348); yylhsminor.yy348 = yymsp[-2].minor.yy348; } -#line 6596 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ ((SStreamOptions*)yymsp[-2].minor.yy342)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy342); yylhsminor.yy342 = yymsp[-2].minor.yy342; } +#line 6656 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 325: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ #line 586 "sql.y" -{ ((SStreamOptions*)yymsp[-2].minor.yy348)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy348); yylhsminor.yy348 = yymsp[-2].minor.yy348; } -#line 6602 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ ((SStreamOptions*)yymsp[-2].minor.yy342)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy342); yylhsminor.yy342 = yymsp[-2].minor.yy342; } +#line 6662 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 326: /* with_meta ::= AS */ #line 591 "sql.y" -{ yymsp[0].minor.yy88 = 0; } -#line 6608 "sql.c" +{ yymsp[0].minor.yy114 = 0; } +#line 6668 "sql.c" break; case 327: /* with_meta ::= WITH META AS */ #line 592 "sql.y" -{ yymsp[-2].minor.yy88 = 1; } -#line 6613 "sql.c" +{ yymsp[-2].minor.yy114 = 1; } +#line 6673 "sql.c" break; case 328: /* with_meta ::= ONLY META AS */ #line 593 "sql.y" -{ yymsp[-2].minor.yy88 = 2; } -#line 6618 "sql.c" +{ yymsp[-2].minor.yy114 = 2; } +#line 6678 "sql.c" break; case 329: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ #line 595 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy345, &yymsp[-2].minor.yy269, yymsp[0].minor.yy348); } -#line 6623 "sql.c" +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy829, &yymsp[-2].minor.yy479, yymsp[0].minor.yy342); } +#line 6683 "sql.c" break; case 330: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ #line 597 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy345, &yymsp[-3].minor.yy269, &yymsp[0].minor.yy269, yymsp[-2].minor.yy88); } -#line 6628 "sql.c" +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy829, &yymsp[-3].minor.yy479, &yymsp[0].minor.yy479, yymsp[-2].minor.yy114); } +#line 6688 "sql.c" break; case 331: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ #line 599 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy345, &yymsp[-4].minor.yy269, yymsp[-1].minor.yy348, yymsp[-3].minor.yy88, yymsp[0].minor.yy348); } -#line 6633 "sql.c" +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy829, &yymsp[-4].minor.yy479, yymsp[-1].minor.yy342, yymsp[-3].minor.yy114, yymsp[0].minor.yy342); } +#line 6693 "sql.c" break; case 332: /* cmd ::= DROP TOPIC exists_opt topic_name */ #line 601 "sql.y" -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy269); } -#line 6638 "sql.c" +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy829, &yymsp[0].minor.yy479); } +#line 6698 "sql.c" break; case 333: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ #line 602 "sql.y" -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy345, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy269); } -#line 6643 "sql.c" +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy829, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy479); } +#line 6703 "sql.c" break; case 334: /* cmd ::= DESC full_table_name */ case 335: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==335); #line 605 "sql.y" -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy348); } -#line 6649 "sql.c" +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy342); } +#line 6709 "sql.c" break; case 336: /* cmd ::= RESET QUERY CACHE */ #line 609 "sql.y" { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } -#line 6654 "sql.c" +#line 6714 "sql.c" break; case 337: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ case 338: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==338); #line 612 "sql.y" -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy345, yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 6660 "sql.c" +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy829, yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 6720 "sql.c" break; case 341: /* explain_options ::= */ #line 620 "sql.y" -{ yymsp[1].minor.yy348 = createDefaultExplainOptions(pCxt); } -#line 6665 "sql.c" +{ yymsp[1].minor.yy342 = createDefaultExplainOptions(pCxt); } +#line 6725 "sql.c" break; case 342: /* explain_options ::= explain_options VERBOSE NK_BOOL */ #line 621 "sql.y" -{ yylhsminor.yy348 = setExplainVerbose(pCxt, yymsp[-2].minor.yy348, &yymsp[0].minor.yy0); } -#line 6670 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setExplainVerbose(pCxt, yymsp[-2].minor.yy342, &yymsp[0].minor.yy0); } +#line 6730 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 343: /* explain_options ::= explain_options RATIO NK_FLOAT */ #line 622 "sql.y" -{ yylhsminor.yy348 = setExplainRatio(pCxt, yymsp[-2].minor.yy348, &yymsp[0].minor.yy0); } -#line 6676 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setExplainRatio(pCxt, yymsp[-2].minor.yy342, &yymsp[0].minor.yy0); } +#line 6736 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 344: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ #line 627 "sql.y" -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy345, yymsp[-9].minor.yy345, &yymsp[-6].minor.yy269, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy720, yymsp[-1].minor.yy88, &yymsp[0].minor.yy269, yymsp[-10].minor.yy345); } -#line 6682 "sql.c" +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy829, yymsp[-9].minor.yy829, &yymsp[-6].minor.yy479, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy750, yymsp[-1].minor.yy114, &yymsp[0].minor.yy479, yymsp[-10].minor.yy829); } +#line 6742 "sql.c" break; case 345: /* cmd ::= DROP FUNCTION exists_opt function_name */ #line 628 "sql.y" -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy269); } -#line 6687 "sql.c" +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy829, &yymsp[0].minor.yy479); } +#line 6747 "sql.c" break; case 350: /* language_opt ::= */ case 384: /* on_vgroup_id ::= */ yytestcase(yyruleno==384); #line 642 "sql.y" -{ yymsp[1].minor.yy269 = nil_token; } -#line 6693 "sql.c" +{ yymsp[1].minor.yy479 = nil_token; } +#line 6753 "sql.c" break; case 351: /* language_opt ::= LANGUAGE NK_STRING */ case 385: /* on_vgroup_id ::= ON NK_INTEGER */ yytestcase(yyruleno==385); #line 643 "sql.y" -{ yymsp[-1].minor.yy269 = yymsp[0].minor.yy0; } -#line 6699 "sql.c" +{ yymsp[-1].minor.yy479 = yymsp[0].minor.yy0; } +#line 6759 "sql.c" break; case 354: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ #line 653 "sql.y" -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy345, &yymsp[-8].minor.yy269, yymsp[-5].minor.yy348, yymsp[-7].minor.yy348, yymsp[-3].minor.yy860, yymsp[-2].minor.yy348, yymsp[0].minor.yy348, yymsp[-4].minor.yy860); } -#line 6704 "sql.c" +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy829, &yymsp[-8].minor.yy479, yymsp[-5].minor.yy342, yymsp[-7].minor.yy342, yymsp[-3].minor.yy20, yymsp[-2].minor.yy342, yymsp[0].minor.yy342, yymsp[-4].minor.yy20); } +#line 6764 "sql.c" break; case 355: /* cmd ::= DROP STREAM exists_opt stream_name */ #line 654 "sql.y" -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy269); } -#line 6709 "sql.c" +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy829, &yymsp[0].minor.yy479); } +#line 6769 "sql.c" break; case 356: /* cmd ::= PAUSE STREAM exists_opt stream_name */ #line 655 "sql.y" -{ pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy269); } -#line 6714 "sql.c" +{ pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy829, &yymsp[0].minor.yy479); } +#line 6774 "sql.c" break; case 357: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ #line 656 "sql.y" -{ pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy345, yymsp[-1].minor.yy345, &yymsp[0].minor.yy269); } -#line 6719 "sql.c" +{ pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy829, yymsp[-1].minor.yy829, &yymsp[0].minor.yy479); } +#line 6779 "sql.c" break; case 364: /* stream_options ::= stream_options TRIGGER AT_ONCE */ case 365: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==365); #line 670 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-2].minor.yy348, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } -#line 6725 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-2].minor.yy342, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } +#line 6785 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 366: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ #line 672 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-3].minor.yy348, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); } -#line 6731 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-3].minor.yy342, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } +#line 6791 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 367: /* stream_options ::= stream_options WATERMARK duration_literal */ #line 673 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-2].minor.yy348, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); } -#line 6737 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-2].minor.yy342, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } +#line 6797 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 368: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ #line 674 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-3].minor.yy348, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } -#line 6743 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-3].minor.yy342, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } +#line 6803 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 369: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ #line 675 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-2].minor.yy348, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } -#line 6749 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-2].minor.yy342, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } +#line 6809 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 370: /* stream_options ::= stream_options DELETE_MARK duration_literal */ #line 676 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-2].minor.yy348, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); } -#line 6755 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-2].minor.yy342, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } +#line 6815 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 371: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ #line 677 "sql.y" -{ yylhsminor.yy348 = setStreamOptions(pCxt, yymsp[-3].minor.yy348, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } -#line 6761 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setStreamOptions(pCxt, yymsp[-3].minor.yy342, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } +#line 6821 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 373: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 571: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==571); - case 592: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==592); + case 571: /* sliding_opt ::= SLIDING NK_LP interval_sliding_duration_literal NK_RP */ yytestcase(yyruleno==571); + case 595: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==595); #line 680 "sql.y" -{ yymsp[-3].minor.yy348 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy348); } -#line 6769 "sql.c" +{ yymsp[-3].minor.yy342 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy342); } +#line 6829 "sql.c" break; case 376: /* cmd ::= KILL CONNECTION NK_INTEGER */ #line 688 "sql.y" { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } -#line 6774 "sql.c" +#line 6834 "sql.c" break; case 377: /* cmd ::= KILL QUERY NK_STRING */ #line 689 "sql.y" { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } -#line 6779 "sql.c" +#line 6839 "sql.c" break; case 378: /* cmd ::= KILL TRANSACTION NK_INTEGER */ #line 690 "sql.y" { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } -#line 6784 "sql.c" +#line 6844 "sql.c" break; case 379: /* cmd ::= BALANCE VGROUP */ #line 693 "sql.y" { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } -#line 6789 "sql.c" +#line 6849 "sql.c" break; case 380: /* cmd ::= BALANCE VGROUP LEADER on_vgroup_id */ #line 694 "sql.y" -{ pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &yymsp[0].minor.yy269); } -#line 6794 "sql.c" +{ pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &yymsp[0].minor.yy479); } +#line 6854 "sql.c" break; case 381: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ #line 695 "sql.y" { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 6799 "sql.c" +#line 6859 "sql.c" break; case 382: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ #line 696 "sql.y" -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy860); } -#line 6804 "sql.c" +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy20); } +#line 6864 "sql.c" break; case 383: /* cmd ::= SPLIT VGROUP NK_INTEGER */ #line 697 "sql.y" { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } -#line 6809 "sql.c" +#line 6869 "sql.c" break; case 386: /* dnode_list ::= DNODE NK_INTEGER */ #line 706 "sql.y" -{ yymsp[-1].minor.yy860 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } -#line 6814 "sql.c" +{ yymsp[-1].minor.yy20 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 6874 "sql.c" break; case 388: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ #line 713 "sql.y" -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 6819 "sql.c" +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 6879 "sql.c" break; case 391: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ #line 722 "sql.y" -{ yymsp[-6].minor.yy348 = createInsertStmt(pCxt, yymsp[-4].minor.yy348, yymsp[-2].minor.yy860, yymsp[0].minor.yy348); } -#line 6824 "sql.c" +{ yymsp[-6].minor.yy342 = createInsertStmt(pCxt, yymsp[-4].minor.yy342, yymsp[-2].minor.yy20, yymsp[0].minor.yy342); } +#line 6884 "sql.c" break; case 392: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ #line 723 "sql.y" -{ yymsp[-3].minor.yy348 = createInsertStmt(pCxt, yymsp[-1].minor.yy348, NULL, yymsp[0].minor.yy348); } -#line 6829 "sql.c" +{ yymsp[-3].minor.yy342 = createInsertStmt(pCxt, yymsp[-1].minor.yy342, NULL, yymsp[0].minor.yy342); } +#line 6889 "sql.c" break; case 393: /* literal ::= NK_INTEGER */ #line 726 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } -#line 6834 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } +#line 6894 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 394: /* literal ::= NK_FLOAT */ #line 727 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } -#line 6840 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } +#line 6900 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 395: /* literal ::= NK_STRING */ #line 728 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } -#line 6846 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } +#line 6906 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 396: /* literal ::= NK_BOOL */ #line 729 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } -#line 6852 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } +#line 6912 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 397: /* literal ::= TIMESTAMP NK_STRING */ #line 730 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } -#line 6858 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } +#line 6918 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 398: /* literal ::= duration_literal */ case 408: /* signed_literal ::= signed */ yytestcase(yyruleno==408); @@ -6874,239 +6934,242 @@ static YYACTIONTYPE yy_reduce( case 526: /* table_reference ::= table_primary */ yytestcase(yyruleno==526); case 527: /* table_reference ::= joined_table */ yytestcase(yyruleno==527); case 531: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==531); - case 594: /* query_simple ::= query_specification */ yytestcase(yyruleno==594); - case 595: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==595); - case 598: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==598); - case 600: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==600); + case 597: /* query_simple ::= query_specification */ yytestcase(yyruleno==597); + case 598: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==598); + case 601: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==601); + case 603: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==603); #line 731 "sql.y" -{ yylhsminor.yy348 = yymsp[0].minor.yy348; } -#line 6883 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = yymsp[0].minor.yy342; } +#line 6943 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 399: /* literal ::= NULL */ #line 732 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } -#line 6889 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } +#line 6949 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 400: /* literal ::= NK_QUESTION */ #line 733 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6895 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 6955 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 401: /* duration_literal ::= NK_VARIABLE */ + case 572: /* interval_sliding_duration_literal ::= NK_VARIABLE */ yytestcase(yyruleno==572); + case 573: /* interval_sliding_duration_literal ::= NK_STRING */ yytestcase(yyruleno==573); + case 574: /* interval_sliding_duration_literal ::= NK_INTEGER */ yytestcase(yyruleno==574); #line 735 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6901 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 6964 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 402: /* signed ::= NK_INTEGER */ #line 737 "sql.y" -{ yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6907 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } +#line 6970 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 403: /* signed ::= NK_PLUS NK_INTEGER */ #line 738 "sql.y" -{ yymsp[-1].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6913 "sql.c" +{ yymsp[-1].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } +#line 6976 "sql.c" break; case 404: /* signed ::= NK_MINUS NK_INTEGER */ #line 739 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -#line 6922 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +#line 6985 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 405: /* signed ::= NK_FLOAT */ #line 744 "sql.y" -{ yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6928 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } +#line 6991 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 406: /* signed ::= NK_PLUS NK_FLOAT */ #line 745 "sql.y" -{ yymsp[-1].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6934 "sql.c" +{ yymsp[-1].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } +#line 6997 "sql.c" break; case 407: /* signed ::= NK_MINUS NK_FLOAT */ #line 746 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } -#line 6943 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +#line 7006 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 409: /* signed_literal ::= NK_STRING */ #line 753 "sql.y" -{ yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } -#line 6949 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +#line 7012 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 410: /* signed_literal ::= NK_BOOL */ #line 754 "sql.y" -{ yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } -#line 6955 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } +#line 7018 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 411: /* signed_literal ::= TIMESTAMP NK_STRING */ #line 755 "sql.y" -{ yymsp[-1].minor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 6961 "sql.c" +{ yymsp[-1].minor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +#line 7024 "sql.c" break; case 412: /* signed_literal ::= duration_literal */ case 414: /* signed_literal ::= literal_func */ yytestcase(yyruleno==414); case 485: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==485); case 551: /* select_item ::= common_expression */ yytestcase(yyruleno==551); case 561: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==561); - case 599: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==599); - case 601: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==601); - case 614: /* search_condition ::= common_expression */ yytestcase(yyruleno==614); + case 602: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==602); + case 604: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==604); + case 617: /* search_condition ::= common_expression */ yytestcase(yyruleno==617); #line 756 "sql.y" -{ yylhsminor.yy348 = releaseRawExprNode(pCxt, yymsp[0].minor.yy348); } -#line 6973 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = releaseRawExprNode(pCxt, yymsp[0].minor.yy342); } +#line 7036 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 413: /* signed_literal ::= NULL */ #line 757 "sql.y" -{ yylhsminor.yy348 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } -#line 6979 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } +#line 7042 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 415: /* signed_literal ::= NK_QUESTION */ #line 759 "sql.y" -{ yylhsminor.yy348 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } -#line 6985 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } +#line 7048 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 431: /* expression ::= pseudo_column */ #line 816 "sql.y" -{ yylhsminor.yy348 = yymsp[0].minor.yy348; setRawExprNodeIsPseudoColumn(pCxt, yylhsminor.yy348, true); } -#line 6991 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = yymsp[0].minor.yy342; setRawExprNodeIsPseudoColumn(pCxt, yylhsminor.yy342, true); } +#line 7054 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 435: /* expression ::= NK_LP expression NK_RP */ case 519: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==519); - case 613: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==613); + case 616: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==616); #line 820 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy348)); } -#line 6999 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy342)); } +#line 7062 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 436: /* expression ::= NK_PLUS expr_or_subquery */ #line 821 "sql.y" { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } -#line 7008 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +#line 7071 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 437: /* expression ::= NK_MINUS expr_or_subquery */ #line 825 "sql.y" { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy348), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy342), NULL)); } -#line 7017 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +#line 7080 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 438: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ #line 829 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7027 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7090 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 439: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ #line 834 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7037 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7100 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 440: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ #line 839 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7047 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7110 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 441: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ #line 844 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7057 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7120 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 442: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ #line 849 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7067 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7130 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 443: /* expression ::= column_reference NK_ARROW NK_STRING */ #line 854 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } -#line 7076 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7139 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 444: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ #line 858 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7086 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7149 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 445: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ #line 863 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7096 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7159 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 448: /* column_reference ::= column_name */ #line 874 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy269, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy269)); } -#line 7102 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy479, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy479)); } +#line 7165 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 449: /* column_reference ::= table_name NK_DOT column_name */ #line 875 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy269, createColumnNode(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy269)); } -#line 7108 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy479, createColumnNode(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy479)); } +#line 7171 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 450: /* pseudo_column ::= ROWTS */ case 451: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==451); @@ -7121,507 +7184,507 @@ static YYACTIONTYPE yy_reduce( case 461: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==461); case 467: /* literal_func ::= NOW */ yytestcase(yyruleno==467); #line 877 "sql.y" -{ yylhsminor.yy348 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } -#line 7125 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } +#line 7188 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 452: /* pseudo_column ::= table_name NK_DOT TBNAME */ #line 879 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy269)))); } -#line 7131 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy479)))); } +#line 7194 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 462: /* function_expression ::= function_name NK_LP expression_list NK_RP */ case 463: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==463); #line 890 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy269, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy269, yymsp[-1].minor.yy860)); } -#line 7138 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy479, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy479, yymsp[-1].minor.yy20)); } +#line 7201 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 464: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ #line 893 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), yymsp[-1].minor.yy720)); } -#line 7144 "sql.c" - yymsp[-5].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), yymsp[-1].minor.yy750)); } +#line 7207 "sql.c" + yymsp[-5].minor.yy342 = yylhsminor.yy342; break; case 466: /* literal_func ::= noarg_func NK_LP NK_RP */ #line 896 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy269, NULL)); } -#line 7150 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy479, NULL)); } +#line 7213 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 481: /* star_func_para_list ::= NK_STAR */ #line 920 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } -#line 7156 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } +#line 7219 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 486: /* star_func_para ::= table_name NK_DOT NK_STAR */ case 554: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==554); #line 929 "sql.y" -{ yylhsminor.yy348 = createColumnNode(pCxt, &yymsp[-2].minor.yy269, &yymsp[0].minor.yy0); } -#line 7163 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createColumnNode(pCxt, &yymsp[-2].minor.yy479, &yymsp[0].minor.yy0); } +#line 7226 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 487: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ #line 932 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy860, yymsp[-1].minor.yy348)); } -#line 7169 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy20, yymsp[-1].minor.yy342)); } +#line 7232 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 488: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ #line 934 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), yymsp[-2].minor.yy860, yymsp[-1].minor.yy348)); } -#line 7175 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), yymsp[-2].minor.yy20, yymsp[-1].minor.yy342)); } +#line 7238 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 491: /* when_then_expr ::= WHEN common_expression THEN common_expression */ #line 941 "sql.y" -{ yymsp[-3].minor.yy348 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348)); } -#line 7181 "sql.c" +{ yymsp[-3].minor.yy342 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342)); } +#line 7244 "sql.c" break; case 493: /* case_when_else_opt ::= ELSE common_expression */ #line 944 "sql.y" -{ yymsp[-1].minor.yy348 = releaseRawExprNode(pCxt, yymsp[0].minor.yy348); } -#line 7186 "sql.c" +{ yymsp[-1].minor.yy342 = releaseRawExprNode(pCxt, yymsp[0].minor.yy342); } +#line 7249 "sql.c" break; case 494: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ case 499: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==499); #line 947 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy696, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy356, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7196 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7259 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 495: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ #line 954 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy348), releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy342), releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7206 "sql.c" - yymsp[-4].minor.yy348 = yylhsminor.yy348; +#line 7269 "sql.c" + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; case 496: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ #line 960 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy348), releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy342), releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7216 "sql.c" - yymsp[-5].minor.yy348 = yylhsminor.yy348; +#line 7279 "sql.c" + yymsp[-5].minor.yy342 = yylhsminor.yy342; break; case 497: /* predicate ::= expr_or_subquery IS NULL */ #line 965 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), NULL)); } -#line 7225 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7288 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 498: /* predicate ::= expr_or_subquery IS NOT NULL */ #line 969 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), NULL)); } -#line 7234 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +#line 7297 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 500: /* compare_op ::= NK_LT */ #line 981 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_LOWER_THAN; } -#line 7240 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_LOWER_THAN; } +#line 7303 "sql.c" break; case 501: /* compare_op ::= NK_GT */ #line 982 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_GREATER_THAN; } -#line 7245 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_GREATER_THAN; } +#line 7308 "sql.c" break; case 502: /* compare_op ::= NK_LE */ #line 983 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_LOWER_EQUAL; } -#line 7250 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_LOWER_EQUAL; } +#line 7313 "sql.c" break; case 503: /* compare_op ::= NK_GE */ #line 984 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_GREATER_EQUAL; } -#line 7255 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_GREATER_EQUAL; } +#line 7318 "sql.c" break; case 504: /* compare_op ::= NK_NE */ #line 985 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_NOT_EQUAL; } -#line 7260 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_NOT_EQUAL; } +#line 7323 "sql.c" break; case 505: /* compare_op ::= NK_EQ */ #line 986 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_EQUAL; } -#line 7265 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_EQUAL; } +#line 7328 "sql.c" break; case 506: /* compare_op ::= LIKE */ #line 987 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_LIKE; } -#line 7270 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_LIKE; } +#line 7333 "sql.c" break; case 507: /* compare_op ::= NOT LIKE */ #line 988 "sql.y" -{ yymsp[-1].minor.yy696 = OP_TYPE_NOT_LIKE; } -#line 7275 "sql.c" +{ yymsp[-1].minor.yy356 = OP_TYPE_NOT_LIKE; } +#line 7338 "sql.c" break; case 508: /* compare_op ::= MATCH */ #line 989 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_MATCH; } -#line 7280 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_MATCH; } +#line 7343 "sql.c" break; case 509: /* compare_op ::= NMATCH */ #line 990 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_NMATCH; } -#line 7285 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_NMATCH; } +#line 7348 "sql.c" break; case 510: /* compare_op ::= CONTAINS */ #line 991 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_JSON_CONTAINS; } -#line 7290 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_JSON_CONTAINS; } +#line 7353 "sql.c" break; case 511: /* in_op ::= IN */ #line 995 "sql.y" -{ yymsp[0].minor.yy696 = OP_TYPE_IN; } -#line 7295 "sql.c" +{ yymsp[0].minor.yy356 = OP_TYPE_IN; } +#line 7358 "sql.c" break; case 512: /* in_op ::= NOT IN */ #line 996 "sql.y" -{ yymsp[-1].minor.yy696 = OP_TYPE_NOT_IN; } -#line 7300 "sql.c" +{ yymsp[-1].minor.yy356 = OP_TYPE_NOT_IN; } +#line 7363 "sql.c" break; case 513: /* in_predicate_value ::= NK_LP literal_list NK_RP */ #line 998 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy860)); } -#line 7305 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy20)); } +#line 7368 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 515: /* boolean_value_expression ::= NOT boolean_primary */ #line 1002 "sql.y" { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy348), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy342), NULL)); } -#line 7314 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +#line 7377 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 516: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ #line 1007 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7324 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7387 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 517: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ #line 1013 "sql.y" { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy348); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy348); - yylhsminor.yy348 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy342); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy342); + yylhsminor.yy342 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } -#line 7334 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +#line 7397 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 525: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ #line 1031 "sql.y" -{ yylhsminor.yy348 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy348, yymsp[0].minor.yy348, NULL); } -#line 7340 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy342, yymsp[0].minor.yy342, NULL); } +#line 7403 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 528: /* table_primary ::= table_name alias_opt */ #line 1037 "sql.y" -{ yylhsminor.yy348 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy269, &yymsp[0].minor.yy269); } -#line 7346 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy479, &yymsp[0].minor.yy479); } +#line 7409 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 529: /* table_primary ::= db_name NK_DOT table_name alias_opt */ #line 1038 "sql.y" -{ yylhsminor.yy348 = createRealTableNode(pCxt, &yymsp[-3].minor.yy269, &yymsp[-1].minor.yy269, &yymsp[0].minor.yy269); } -#line 7352 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createRealTableNode(pCxt, &yymsp[-3].minor.yy479, &yymsp[-1].minor.yy479, &yymsp[0].minor.yy479); } +#line 7415 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; case 530: /* table_primary ::= subquery alias_opt */ #line 1039 "sql.y" -{ yylhsminor.yy348 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy348), &yymsp[0].minor.yy269); } -#line 7358 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy342), &yymsp[0].minor.yy479); } +#line 7421 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 532: /* alias_opt ::= */ #line 1044 "sql.y" -{ yymsp[1].minor.yy269 = nil_token; } -#line 7364 "sql.c" +{ yymsp[1].minor.yy479 = nil_token; } +#line 7427 "sql.c" break; case 534: /* alias_opt ::= AS table_alias */ #line 1046 "sql.y" -{ yymsp[-1].minor.yy269 = yymsp[0].minor.yy269; } -#line 7369 "sql.c" +{ yymsp[-1].minor.yy479 = yymsp[0].minor.yy479; } +#line 7432 "sql.c" break; case 535: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ case 536: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==536); #line 1048 "sql.y" -{ yymsp[-2].minor.yy348 = yymsp[-1].minor.yy348; } -#line 7375 "sql.c" +{ yymsp[-2].minor.yy342 = yymsp[-1].minor.yy342; } +#line 7438 "sql.c" break; case 537: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ #line 1053 "sql.y" -{ yylhsminor.yy348 = createJoinTableNode(pCxt, yymsp[-4].minor.yy184, yymsp[-5].minor.yy348, yymsp[-2].minor.yy348, yymsp[0].minor.yy348); } -#line 7380 "sql.c" - yymsp[-5].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createJoinTableNode(pCxt, yymsp[-4].minor.yy392, yymsp[-5].minor.yy342, yymsp[-2].minor.yy342, yymsp[0].minor.yy342); } +#line 7443 "sql.c" + yymsp[-5].minor.yy342 = yylhsminor.yy342; break; case 538: /* join_type ::= */ #line 1057 "sql.y" -{ yymsp[1].minor.yy184 = JOIN_TYPE_INNER; } -#line 7386 "sql.c" +{ yymsp[1].minor.yy392 = JOIN_TYPE_INNER; } +#line 7449 "sql.c" break; case 539: /* join_type ::= INNER */ #line 1058 "sql.y" -{ yymsp[0].minor.yy184 = JOIN_TYPE_INNER; } -#line 7391 "sql.c" +{ yymsp[0].minor.yy392 = JOIN_TYPE_INNER; } +#line 7454 "sql.c" break; case 540: /* query_specification ::= SELECT hint_list set_quantifier_opt tag_mode_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ #line 1064 "sql.y" { - yymsp[-13].minor.yy348 = createSelectStmt(pCxt, yymsp[-11].minor.yy345, yymsp[-9].minor.yy860, yymsp[-8].minor.yy348, yymsp[-12].minor.yy860); - yymsp[-13].minor.yy348 = setSelectStmtTagMode(pCxt, yymsp[-13].minor.yy348, yymsp[-10].minor.yy345); - yymsp[-13].minor.yy348 = addWhereClause(pCxt, yymsp[-13].minor.yy348, yymsp[-7].minor.yy348); - yymsp[-13].minor.yy348 = addPartitionByClause(pCxt, yymsp[-13].minor.yy348, yymsp[-6].minor.yy860); - yymsp[-13].minor.yy348 = addWindowClauseClause(pCxt, yymsp[-13].minor.yy348, yymsp[-2].minor.yy348); - yymsp[-13].minor.yy348 = addGroupByClause(pCxt, yymsp[-13].minor.yy348, yymsp[-1].minor.yy860); - yymsp[-13].minor.yy348 = addHavingClause(pCxt, yymsp[-13].minor.yy348, yymsp[0].minor.yy348); - yymsp[-13].minor.yy348 = addRangeClause(pCxt, yymsp[-13].minor.yy348, yymsp[-5].minor.yy348); - yymsp[-13].minor.yy348 = addEveryClause(pCxt, yymsp[-13].minor.yy348, yymsp[-4].minor.yy348); - yymsp[-13].minor.yy348 = addFillClause(pCxt, yymsp[-13].minor.yy348, yymsp[-3].minor.yy348); + yymsp[-13].minor.yy342 = createSelectStmt(pCxt, yymsp[-11].minor.yy829, yymsp[-9].minor.yy20, yymsp[-8].minor.yy342, yymsp[-12].minor.yy20); + yymsp[-13].minor.yy342 = setSelectStmtTagMode(pCxt, yymsp[-13].minor.yy342, yymsp[-10].minor.yy829); + yymsp[-13].minor.yy342 = addWhereClause(pCxt, yymsp[-13].minor.yy342, yymsp[-7].minor.yy342); + yymsp[-13].minor.yy342 = addPartitionByClause(pCxt, yymsp[-13].minor.yy342, yymsp[-6].minor.yy20); + yymsp[-13].minor.yy342 = addWindowClauseClause(pCxt, yymsp[-13].minor.yy342, yymsp[-2].minor.yy342); + yymsp[-13].minor.yy342 = addGroupByClause(pCxt, yymsp[-13].minor.yy342, yymsp[-1].minor.yy20); + yymsp[-13].minor.yy342 = addHavingClause(pCxt, yymsp[-13].minor.yy342, yymsp[0].minor.yy342); + yymsp[-13].minor.yy342 = addRangeClause(pCxt, yymsp[-13].minor.yy342, yymsp[-5].minor.yy342); + yymsp[-13].minor.yy342 = addEveryClause(pCxt, yymsp[-13].minor.yy342, yymsp[-4].minor.yy342); + yymsp[-13].minor.yy342 = addFillClause(pCxt, yymsp[-13].minor.yy342, yymsp[-3].minor.yy342); } -#line 7407 "sql.c" +#line 7470 "sql.c" break; case 541: /* hint_list ::= */ #line 1079 "sql.y" -{ yymsp[1].minor.yy860 = createHintNodeList(pCxt, NULL); } -#line 7412 "sql.c" +{ yymsp[1].minor.yy20 = createHintNodeList(pCxt, NULL); } +#line 7475 "sql.c" break; case 542: /* hint_list ::= NK_HINT */ #line 1080 "sql.y" -{ yylhsminor.yy860 = createHintNodeList(pCxt, &yymsp[0].minor.yy0); } -#line 7417 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; +{ yylhsminor.yy20 = createHintNodeList(pCxt, &yymsp[0].minor.yy0); } +#line 7480 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; case 547: /* set_quantifier_opt ::= ALL */ #line 1091 "sql.y" -{ yymsp[0].minor.yy345 = false; } -#line 7423 "sql.c" +{ yymsp[0].minor.yy829 = false; } +#line 7486 "sql.c" break; case 550: /* select_item ::= NK_STAR */ #line 1098 "sql.y" -{ yylhsminor.yy348 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } -#line 7428 "sql.c" - yymsp[0].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } +#line 7491 "sql.c" + yymsp[0].minor.yy342 = yylhsminor.yy342; break; case 552: /* select_item ::= common_expression column_alias */ case 562: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==562); #line 1100 "sql.y" -{ yylhsminor.yy348 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy348), &yymsp[0].minor.yy269); } -#line 7435 "sql.c" - yymsp[-1].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy342), &yymsp[0].minor.yy479); } +#line 7498 "sql.c" + yymsp[-1].minor.yy342 = yylhsminor.yy342; break; case 553: /* select_item ::= common_expression AS column_alias */ case 563: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==563); #line 1101 "sql.y" -{ yylhsminor.yy348 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), &yymsp[0].minor.yy269); } -#line 7442 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; +{ yylhsminor.yy342 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), &yymsp[0].minor.yy479); } +#line 7505 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; break; case 558: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 583: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==583); - case 603: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==603); + case 586: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==586); + case 606: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==606); #line 1110 "sql.y" -{ yymsp[-2].minor.yy860 = yymsp[0].minor.yy860; } -#line 7450 "sql.c" +{ yymsp[-2].minor.yy20 = yymsp[0].minor.yy20; } +#line 7513 "sql.c" break; - case 565: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + case 565: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA interval_sliding_duration_literal NK_RP */ #line 1123 "sql.y" -{ yymsp[-5].minor.yy348 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), releaseRawExprNode(pCxt, yymsp[-1].minor.yy348)); } -#line 7455 "sql.c" +{ yymsp[-5].minor.yy342 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), releaseRawExprNode(pCxt, yymsp[-1].minor.yy342)); } +#line 7518 "sql.c" break; case 566: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ #line 1124 "sql.y" -{ yymsp[-3].minor.yy348 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy348)); } -#line 7460 "sql.c" +{ yymsp[-3].minor.yy342 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy342)); } +#line 7523 "sql.c" break; - case 567: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + case 567: /* twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ #line 1126 "sql.y" -{ yymsp[-5].minor.yy348 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), NULL, yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 7465 "sql.c" +{ yymsp[-5].minor.yy342 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), NULL, yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 7528 "sql.c" break; - case 568: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -#line 1129 "sql.y" -{ yymsp[-7].minor.yy348 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy348), releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), yymsp[-1].minor.yy348, yymsp[0].minor.yy348); } -#line 7470 "sql.c" + case 568: /* twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_COMMA interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ +#line 1130 "sql.y" +{ yymsp[-7].minor.yy342 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy342), releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), yymsp[-1].minor.yy342, yymsp[0].minor.yy342); } +#line 7533 "sql.c" break; case 569: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -#line 1131 "sql.y" -{ yymsp[-6].minor.yy348 = createEventWindowNode(pCxt, yymsp[-3].minor.yy348, yymsp[0].minor.yy348); } -#line 7475 "sql.c" +#line 1132 "sql.y" +{ yymsp[-6].minor.yy342 = createEventWindowNode(pCxt, yymsp[-3].minor.yy342, yymsp[0].minor.yy342); } +#line 7538 "sql.c" break; - case 573: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -#line 1137 "sql.y" -{ yymsp[-3].minor.yy348 = createFillNode(pCxt, yymsp[-1].minor.yy758, NULL); } -#line 7480 "sql.c" + case 576: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +#line 1142 "sql.y" +{ yymsp[-3].minor.yy342 = createFillNode(pCxt, yymsp[-1].minor.yy590, NULL); } +#line 7543 "sql.c" break; - case 574: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ -#line 1138 "sql.y" -{ yymsp[-5].minor.yy348 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy860)); } -#line 7485 "sql.c" - break; - case 575: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ -#line 1139 "sql.y" -{ yymsp[-5].minor.yy348 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy860)); } -#line 7490 "sql.c" - break; - case 576: /* fill_mode ::= NONE */ + case 577: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ #line 1143 "sql.y" -{ yymsp[0].minor.yy758 = FILL_MODE_NONE; } -#line 7495 "sql.c" +{ yymsp[-5].minor.yy342 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy20)); } +#line 7548 "sql.c" break; - case 577: /* fill_mode ::= PREV */ + case 578: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ #line 1144 "sql.y" -{ yymsp[0].minor.yy758 = FILL_MODE_PREV; } -#line 7500 "sql.c" +{ yymsp[-5].minor.yy342 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy20)); } +#line 7553 "sql.c" break; - case 578: /* fill_mode ::= NULL */ -#line 1145 "sql.y" -{ yymsp[0].minor.yy758 = FILL_MODE_NULL; } -#line 7505 "sql.c" - break; - case 579: /* fill_mode ::= NULL_F */ -#line 1146 "sql.y" -{ yymsp[0].minor.yy758 = FILL_MODE_NULL_F; } -#line 7510 "sql.c" - break; - case 580: /* fill_mode ::= LINEAR */ -#line 1147 "sql.y" -{ yymsp[0].minor.yy758 = FILL_MODE_LINEAR; } -#line 7515 "sql.c" - break; - case 581: /* fill_mode ::= NEXT */ + case 579: /* fill_mode ::= NONE */ #line 1148 "sql.y" -{ yymsp[0].minor.yy758 = FILL_MODE_NEXT; } -#line 7520 "sql.c" +{ yymsp[0].minor.yy590 = FILL_MODE_NONE; } +#line 7558 "sql.c" break; - case 584: /* group_by_list ::= expr_or_subquery */ -#line 1157 "sql.y" -{ yylhsminor.yy860 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); } -#line 7525 "sql.c" - yymsp[0].minor.yy860 = yylhsminor.yy860; - break; - case 585: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -#line 1158 "sql.y" -{ yylhsminor.yy860 = addNodeToList(pCxt, yymsp[-2].minor.yy860, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy348))); } -#line 7531 "sql.c" - yymsp[-2].minor.yy860 = yylhsminor.yy860; - break; - case 589: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -#line 1165 "sql.y" -{ yymsp[-5].minor.yy348 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy348), releaseRawExprNode(pCxt, yymsp[-1].minor.yy348)); } -#line 7537 "sql.c" - break; - case 590: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ -#line 1167 "sql.y" -{ yymsp[-3].minor.yy348 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy348)); } -#line 7542 "sql.c" - break; - case 593: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ -#line 1174 "sql.y" -{ - yylhsminor.yy348 = addOrderByClause(pCxt, yymsp[-3].minor.yy348, yymsp[-2].minor.yy860); - yylhsminor.yy348 = addSlimitClause(pCxt, yylhsminor.yy348, yymsp[-1].minor.yy348); - yylhsminor.yy348 = addLimitClause(pCxt, yylhsminor.yy348, yymsp[0].minor.yy348); - } -#line 7551 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; - break; - case 596: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -#line 1184 "sql.y" -{ yylhsminor.yy348 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy348, yymsp[0].minor.yy348); } -#line 7557 "sql.c" - yymsp[-3].minor.yy348 = yylhsminor.yy348; - break; - case 597: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -#line 1186 "sql.y" -{ yylhsminor.yy348 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy348, yymsp[0].minor.yy348); } + case 580: /* fill_mode ::= PREV */ +#line 1149 "sql.y" +{ yymsp[0].minor.yy590 = FILL_MODE_PREV; } #line 7563 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; break; - case 605: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 609: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==609); -#line 1200 "sql.y" -{ yymsp[-1].minor.yy348 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 7570 "sql.c" + case 581: /* fill_mode ::= NULL */ +#line 1150 "sql.y" +{ yymsp[0].minor.yy590 = FILL_MODE_NULL; } +#line 7568 "sql.c" break; - case 606: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 610: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==610); -#line 1201 "sql.y" -{ yymsp[-3].minor.yy348 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } -#line 7576 "sql.c" + case 582: /* fill_mode ::= NULL_F */ +#line 1151 "sql.y" +{ yymsp[0].minor.yy590 = FILL_MODE_NULL_F; } +#line 7573 "sql.c" break; - case 607: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 611: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==611); -#line 1202 "sql.y" -{ yymsp[-3].minor.yy348 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } -#line 7582 "sql.c" + case 583: /* fill_mode ::= LINEAR */ +#line 1152 "sql.y" +{ yymsp[0].minor.yy590 = FILL_MODE_LINEAR; } +#line 7578 "sql.c" break; - case 612: /* subquery ::= NK_LP query_expression NK_RP */ -#line 1210 "sql.y" -{ yylhsminor.yy348 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy348); } -#line 7587 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; + case 584: /* fill_mode ::= NEXT */ +#line 1153 "sql.y" +{ yymsp[0].minor.yy590 = FILL_MODE_NEXT; } +#line 7583 "sql.c" break; - case 617: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -#line 1224 "sql.y" -{ yylhsminor.yy348 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy348), yymsp[-1].minor.yy870, yymsp[0].minor.yy841); } -#line 7593 "sql.c" - yymsp[-2].minor.yy348 = yylhsminor.yy348; + case 587: /* group_by_list ::= expr_or_subquery */ +#line 1162 "sql.y" +{ yylhsminor.yy20 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } +#line 7588 "sql.c" + yymsp[0].minor.yy20 = yylhsminor.yy20; break; - case 618: /* ordering_specification_opt ::= */ -#line 1228 "sql.y" -{ yymsp[1].minor.yy870 = ORDER_ASC; } -#line 7599 "sql.c" + case 588: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +#line 1163 "sql.y" +{ yylhsminor.yy20 = addNodeToList(pCxt, yymsp[-2].minor.yy20, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy342))); } +#line 7594 "sql.c" + yymsp[-2].minor.yy20 = yylhsminor.yy20; break; - case 619: /* ordering_specification_opt ::= ASC */ -#line 1229 "sql.y" -{ yymsp[0].minor.yy870 = ORDER_ASC; } -#line 7604 "sql.c" + case 592: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +#line 1170 "sql.y" +{ yymsp[-5].minor.yy342 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy342), releaseRawExprNode(pCxt, yymsp[-1].minor.yy342)); } +#line 7600 "sql.c" break; - case 620: /* ordering_specification_opt ::= DESC */ -#line 1230 "sql.y" -{ yymsp[0].minor.yy870 = ORDER_DESC; } -#line 7609 "sql.c" + case 593: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ +#line 1172 "sql.y" +{ yymsp[-3].minor.yy342 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy342)); } +#line 7605 "sql.c" break; - case 621: /* null_ordering_opt ::= */ -#line 1234 "sql.y" -{ yymsp[1].minor.yy841 = NULL_ORDER_DEFAULT; } + case 596: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ +#line 1179 "sql.y" +{ + yylhsminor.yy342 = addOrderByClause(pCxt, yymsp[-3].minor.yy342, yymsp[-2].minor.yy20); + yylhsminor.yy342 = addSlimitClause(pCxt, yylhsminor.yy342, yymsp[-1].minor.yy342); + yylhsminor.yy342 = addLimitClause(pCxt, yylhsminor.yy342, yymsp[0].minor.yy342); + } #line 7614 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; - case 622: /* null_ordering_opt ::= NULLS FIRST */ + case 599: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +#line 1189 "sql.y" +{ yylhsminor.yy342 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy342, yymsp[0].minor.yy342); } +#line 7620 "sql.c" + yymsp[-3].minor.yy342 = yylhsminor.yy342; + break; + case 600: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +#line 1191 "sql.y" +{ yylhsminor.yy342 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy342, yymsp[0].minor.yy342); } +#line 7626 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; + break; + case 608: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 612: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==612); +#line 1205 "sql.y" +{ yymsp[-1].minor.yy342 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } +#line 7633 "sql.c" + break; + case 609: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 613: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==613); +#line 1206 "sql.y" +{ yymsp[-3].minor.yy342 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } +#line 7639 "sql.c" + break; + case 610: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 614: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==614); +#line 1207 "sql.y" +{ yymsp[-3].minor.yy342 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } +#line 7645 "sql.c" + break; + case 615: /* subquery ::= NK_LP query_expression NK_RP */ +#line 1215 "sql.y" +{ yylhsminor.yy342 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy342); } +#line 7650 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; + break; + case 620: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +#line 1229 "sql.y" +{ yylhsminor.yy342 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy342), yymsp[-1].minor.yy592, yymsp[0].minor.yy689); } +#line 7656 "sql.c" + yymsp[-2].minor.yy342 = yylhsminor.yy342; + break; + case 621: /* ordering_specification_opt ::= */ +#line 1233 "sql.y" +{ yymsp[1].minor.yy592 = ORDER_ASC; } +#line 7662 "sql.c" + break; + case 622: /* ordering_specification_opt ::= ASC */ +#line 1234 "sql.y" +{ yymsp[0].minor.yy592 = ORDER_ASC; } +#line 7667 "sql.c" + break; + case 623: /* ordering_specification_opt ::= DESC */ #line 1235 "sql.y" -{ yymsp[-1].minor.yy841 = NULL_ORDER_FIRST; } -#line 7619 "sql.c" +{ yymsp[0].minor.yy592 = ORDER_DESC; } +#line 7672 "sql.c" break; - case 623: /* null_ordering_opt ::= NULLS LAST */ -#line 1236 "sql.y" -{ yymsp[-1].minor.yy841 = NULL_ORDER_LAST; } -#line 7624 "sql.c" + case 624: /* null_ordering_opt ::= */ +#line 1239 "sql.y" +{ yymsp[1].minor.yy689 = NULL_ORDER_DEFAULT; } +#line 7677 "sql.c" + break; + case 625: /* null_ordering_opt ::= NULLS FIRST */ +#line 1240 "sql.y" +{ yymsp[-1].minor.yy689 = NULL_ORDER_FIRST; } +#line 7682 "sql.c" + break; + case 626: /* null_ordering_opt ::= NULLS LAST */ +#line 1241 "sql.y" +{ yymsp[-1].minor.yy689 = NULL_ORDER_LAST; } +#line 7687 "sql.c" break; default: break; @@ -7694,7 +7757,7 @@ static void yy_syntax_error( } else if (TSDB_CODE_PAR_DB_NOT_SPECIFIED == pCxt->errCode && TK_NK_FLOAT == TOKEN.type) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } -#line 7697 "sql.c" +#line 7760 "sql.c" /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE @@ -7780,56 +7843,12 @@ void Parse( } #endif - while(1){ /* Exit by "break" */ - assert( yypParser->yytos>=yypParser->yystack ); + do{ assert( yyact==yypParser->yytos->stateno ); yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ - unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ -#ifndef NDEBUG - assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); - if( yyTraceFILE ){ - int yysize = yyRuleInfoNRhs[yyruleno]; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyrulenoyytos[yysize].stateno); - }else{ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", - yyTracePrompt, yyruleno, yyRuleName[yyruleno], - yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == - (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - break; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - break; - } - } -#endif - } - yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM); + yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, + yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY @@ -7885,13 +7904,14 @@ void Parse( yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); yymajor = YYNOCODE; }else{ - while( yypParser->yytos > yypParser->yystack ){ - yyact = yy_find_reduce_action(yypParser->yytos->stateno, - YYERRORSYMBOL); - if( yyact<=YY_MAX_SHIFTREDUCE ) break; + while( yypParser->yytos >= yypParser->yystack + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + ){ yy_pop_parser_stack(yypParser); } - if( yypParser->yytos <= yypParser->yystack || yymajor==0 ){ + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); #ifndef YYNOERRORRECOVERY @@ -7941,7 +7961,7 @@ void Parse( break; #endif } - } + }while( yypParser->yytos>yypParser->yystack ); #ifndef NDEBUG if( yyTraceFILE ){ yyStackEntry *i; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 33d6a329ee..d7b3f51961 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -379,7 +379,7 @@ static int32_t stbSplRewriteFuns(const SNodeList* pFuncs, SNodeList** pPartialFu return TSDB_CODE_SUCCESS; } -static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { +static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex, uint8_t precision) { int32_t index = 0; SNode* pFunc = NULL; FOREACH(pFunc, pFuncs) { @@ -400,6 +400,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { int32_t len = snprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); taosCreateMD5Hash(name, len); strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + pWStart->node.resType.precision = precision; int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { @@ -466,7 +467,7 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic int32_t index = 0; int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = stbSplAppendWStart(pPartWin->pFuncs, &index); + code = stbSplAppendWStart(pPartWin->pFuncs, &index, ((SColumnNode*)pMergeWindow->pTspk)->node.resType.precision); } if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pPartWin->pFuncs, &pPartWin->node.pTargets); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8b0451604c..fcd39092bd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -49,6 +49,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 2 diff --git a/tests/system-test/2-query/interval_unit.py b/tests/system-test/2-query/interval_unit.py new file mode 100644 index 0000000000..9430c74cff --- /dev/null +++ b/tests/system-test/2-query/interval_unit.py @@ -0,0 +1,197 @@ +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d', precision: str='ms'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName), 1) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s precision '%s'"%(dbName, vgroups, replica, duration, precision), 1) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString, 1) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString, 1) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName, 1) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql, 1) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql, 1) + tdLog.debug("insert data ............ [OK]") + return + + def prepare_db_for_interval_unit_test(self, dbname: str, precision: str = 'ms', startTs = 1537146000000): + self.create_database(tdSql, dbname, precision=precision) + paraDict = {'dbName': dbname, + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, + {'type': 'BIGINT', 'count':1}, + {'type': 'FLOAT', 'count':1}, + {'type': 'DOUBLE', 'count':1}, + {'type': 'smallint', 'count':1}, + {'type': 'tinyint', 'count':1}, + {'type': 'bool', 'count':1}, + {'type': 'binary', 'len':10, 'count':1}, + {'type': 'nchar', 'len':10, 'count':1} ], + 'tagSchema': [{'type': 'INT', 'count':1}, + {'type': 'nchar', 'len':20, 'count':1}, + {'type': 'binary', 'len':20, 'count':1}, + {'type': 'BIGINT', 'count':1}, + {'type': 'smallint', 'count':1}, + {'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 3000, + 'startTs': startTs, + 'tsStep': 100} + + self.create_stable(tdSql, paraDict) + + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + + def prepare_for_interval_unit_test(self): + self.prepare_db_for_interval_unit_test('msdb', 'ms', startTs=1600000000000) + self.prepare_db_for_interval_unit_test('usdb', 'us', startTs=1600000000000000) + self.prepare_db_for_interval_unit_test('nsdb', 'ns', startTs=1600000000000000000) + + def check_explain_res_has_row(self, plan_str_expect: str, rows): + plan_found = False + for row in rows: + if str(row).find(plan_str_expect) >= 0: + tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row))) + plan_found = True + break + if not plan_found: + tdLog.exit("plan: %s not found in res: [%s]" % (plan_str_expect, str(rows))) + + def explain_and_assert_res(self, sql, expect): + tdSql.query(sql) + res = tdSql.queryResult + self.check_explain_res_has_row(expect, res) + + def test_interval_normal_cases(self): + tdSql.execute('use msdb') + sql_template = 'explain verbose true select count(*), min(c1) from meters interval(%s) sliding(%s)' + + fail_durations = ["'1 s'", "'11ns'", "'11ms'", "'11+2s'", + "' 112s '", '"100s "', "' 112 s '", "'112a20s'", "'s'", + '"1 s"', '"1"', '"100"', '"1y"', '1y', '"1s""', "'12s'12s'", "'12s\""] + + for dura in fail_durations: + tdSql.error(sql_template % (dura, dura)) + + msdb_success_durations = ['1s', '"1s"', '"1000000a"', "'100d'", + "'10m'", '"10h"', '1000'] + + for dura in msdb_success_durations: + tdSql.query(sql_template % (dura, dura), queryTimes=1) + unit = '' + if dura.isdigit(): + unit = 'a' + self.check_explain_res_has_row('interval=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult) + self.check_explain_res_has_row('sliding=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult) + + usdb_success_durations = ['"1s"', "'10s'", '"10d"', '"1000000u"', '1000000'] + for dura in usdb_success_durations: + tdSql.execute("use usdb") + tdSql.query(sql_template % (dura, dura), queryTimes=1) + unit = '' + if dura.isdigit(): + unit = 'u' + self.check_explain_res_has_row('interval=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult) + self.check_explain_res_has_row('sliding=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult) + + nsdb_success_durations = ['"1s"', "'10s'", '"10d"', '"1000000u"', '"1000000000b"', '1000000000'] + for dura in usdb_success_durations: + tdSql.execute("use nsdb") + tdSql.query(sql_template % (dura, dura), queryTimes=1) + unit = '' + if dura.isdigit(): + unit = 'b' + self.check_explain_res_has_row('interval=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult) + self.check_explain_res_has_row('sliding=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult) + + def test_interval_unit_feature(self): + self.prepare_for_interval_unit_test() + self.test_interval_normal_cases() + + def run(self): + self.test_interval_unit_feature() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 28b69d279ff96c88b72a717a08137a0a88925a5f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Oct 2023 18:28:52 +0800 Subject: [PATCH 149/177] fix:add except characters for smlAutoChildTableNameDelimiter --- docs/en/14-reference/13-schemaless/13-schemaless.md | 2 +- docs/zh/14-reference/13-schemaless/13-schemaless.md | 2 +- tests/pytest/util/dnodes.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 11d3c0d0ab..eb336f4633 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -94,7 +94,7 @@ The string's MD5 hash value "md5_val" is calculated after the ranking is complet ::: If you do not want to use an automatically generated table name, there are two ways to specify sub table names, the first one has a higher priority. -You can configure smlAutoChildTableNameDelimiter in taos.cfg, for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. +You can configure smlAutoChildTableNameDelimiter in taos.cfg(except for `@ # space \r \t \n`), for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 084e31d810..723531676c 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -96,7 +96,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 :::tip 如果不想用自动生成的表名，有两种指定子表名的方式，第一种优先级更高： -通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定。 +通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定（`@ # 空格 回车 换行 制表符`除外)。 举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 通过在taos.cfg里配置 smlChildTableName 参数来指定。 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c4fc1ce654..5f1522bfd9 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -64,7 +64,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) + cmd = "echo %s '%s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) From 959f8105ee7a641dc9ede2d9487b73f45b5614a0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 19 Oct 2023 18:55:12 +0800 Subject: [PATCH 150/177] enh: require command to trigger repairing vnodes on replaced disks --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 8 ++++---- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 4 ++-- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 4 ++-- source/dnode/vnode/src/sma/smaOpen.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbOpen.c | 7 ++++++- source/dnode/vnode/src/vnd/vnodeOpen.c | 8 ++++---- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 3d7f2b9e9e..cc542f51ce 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -305,7 +305,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { goto _OVER; } - SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, true); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr()); code = terrno; @@ -452,7 +452,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("vgId:%d, begin to open vnode", vgId); - SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); return -1; @@ -602,7 +602,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("vgId:%d, open vnode", dstVgId); - SVnode *pImpl = vnodeOpen(dstPath, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(dstPath, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", dstVgId, dstPath, terrstr()); @@ -707,7 +707,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("vgId:%d, begin to open vnode", vgId); - SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); return -1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 973c45eda7..d2093ff77c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -272,11 +272,11 @@ static void *vmOpenVnodeInThread(void *param) { int32_t diskPrimary = pCfg->diskPrimary; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); - SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d since %s", pCfg->vgId, pThread->threadIndex, terrstr()); - if (terrno != TSDB_CODE_VND_NOT_EXIST) { + if (terrno != TSDB_CODE_NEED_RETRY) { pThread->failed++; continue; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index a120ecf9db..6a0c991be4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -58,7 +58,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, int32_t diskPrimary, STfs *pTfs); void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs); -SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb); +SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force); void vnodePreClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode); void vnodeSyncCheckTimeout(SVnode *pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 68334b3b63..12e273c32d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -202,7 +202,7 @@ typedef struct SMetaInfo { int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader); // tsdb -int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); +int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb); @@ -267,7 +267,7 @@ int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg); // sma int32_t smaInit(); void smaCleanUp(); -int32_t smaOpen(SVnode* pVnode, int8_t rollback); +int32_t smaOpen(SVnode* pVnode, int8_t rollback, bool force); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); int32_t smaPrepareAsyncCommit(SSma* pSma); diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 09929d138e..49f25c0b0a 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -30,7 +30,7 @@ static int32_t rsmaRestore(SSma *pSma); pKeepCfg->keepTimeOffset = 0; \ } while (0) -#define SMA_OPEN_RSMA_IMPL(v, l) \ +#define SMA_OPEN_RSMA_IMPL(v, l, force) \ do { \ SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ if (!RETENTION_VALID(r)) { \ @@ -42,7 +42,7 @@ static int32_t rsmaRestore(SSma *pSma); } \ code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ TSDB_CHECK_CODE(code, lino, _exit); \ - if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback) < 0) { \ + if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force) < 0) { \ code = terrno; \ TSDB_CHECK_CODE(code, lino, _exit); \ } \ @@ -118,7 +118,7 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty return terrno; } -int32_t smaOpen(SVnode *pVnode, int8_t rollback) { +int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) { int32_t code = 0; int32_t lino = 0; STsdbCfg *pCfg = &pVnode->config.tsdbCfg; @@ -139,11 +139,11 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback) { STsdbKeepCfg keepCfg = {0}; for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (i == TSDB_RETENTION_L0) { - SMA_OPEN_RSMA_IMPL(pVnode, 0); + SMA_OPEN_RSMA_IMPL(pVnode, 0, force); } else if (i == TSDB_RETENTION_L1) { - SMA_OPEN_RSMA_IMPL(pVnode, 1); + SMA_OPEN_RSMA_IMPL(pVnode, 1, force); } else if (i == TSDB_RETENTION_L2) { - SMA_OPEN_RSMA_IMPL(pVnode, 2); + SMA_OPEN_RSMA_IMPL(pVnode, 2, force); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index b060edbd91..6dd66c7a40 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -35,7 +35,7 @@ int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { * @param dir * @return int */ -int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback) { +int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) { STsdb *pTsdb = NULL; int slen = 0; @@ -72,6 +72,11 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee goto _err; } + if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) { + terrno = TSDB_CODE_NEED_RETRY; + goto _err; + } + if (tsdbOpenCache(pTsdb) < 0) { goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 7ba542cbf1..3bdecee79b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -330,7 +330,7 @@ static int32_t vnodeCheckDisk(int32_t diskPrimary, STfs *pTfs) { return 0; } -SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb) { +SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force) { SVnode *pVnode = NULL; SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; @@ -350,7 +350,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC ret = vnodeLoadInfo(dir, &info); if (ret < 0) { vError("failed to open vnode from %s since %s", path, tstrerror(terrno)); - terrno = TSDB_CODE_VND_NOT_EXIST; + terrno = TSDB_CODE_NEED_RETRY; return NULL; } @@ -419,7 +419,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open tsdb - if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback) < 0) { + if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback, force) < 0) { vError("vgId:%d, failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -453,7 +453,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open sma - if (smaOpen(pVnode, rollback)) { + if (smaOpen(pVnode, rollback, force)) { vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } From 369586a5c6291346c5f9b667f96fe55d526991f6 Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Thu, 19 Oct 2023 20:33:31 +0800 Subject: [PATCH 151/177] fix auto.offset.reset and snapshot.enable --- docs/en/07-develop/07-tmq.mdx | 10 +++++----- docs/en/14-reference/03-connector/04-java.mdx | 6 +++--- .../src/main/java/com/taos/example/SubscribeDemo.java | 1 - .../java/com/taos/example/WebsocketSubscribeDemo.java | 1 - docs/examples/python/tmq_assignment_example.py | 3 --- .../examples/python/tmq_websocket_assgnment_example.py | 3 --- docs/zh/07-develop/{07-tmq.md => 07-tmq.mdx} | 10 +++++----- docs/zh/08-connector/14-java.mdx | 9 ++++----- 8 files changed, 17 insertions(+), 26 deletions(-) rename docs/zh/07-develop/{07-tmq.md => 07-tmq.mdx} (99%) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 7dd06c9ca2..8e7bbf22e9 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -372,7 +372,7 @@ tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); tmq_conf_set(conf, "group.id", "cgrpName"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); -tmq_conf_set(conf, "auto.offset.reset", "earliest"); +tmq_conf_set(conf, "auto.offset.reset", "latest"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -402,7 +402,7 @@ properties.setProperty("group.id", "cgrpName"); properties.setProperty("bootstrap.servers", "127.0.0.1:6030"); properties.setProperty("td.connect.user", "root"); properties.setProperty("td.connect.pass", "taosdata"); -properties.setProperty("auto.offset.reset", "earliest"); +properties.setProperty("auto.offset.reset", "latest"); properties.setProperty("msg.with.table.name", "true"); properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer"); @@ -442,7 +442,7 @@ consumer, err := NewConsumer(conf) let mut dsn: Dsn = "taos://".parse()?; dsn.set("group.id", "group1"); dsn.set("client.id", "test"); -dsn.set("auto.offset.reset", "earliest"); +dsn.set("auto.offset.reset", "latest"); let tmq = TmqBuilder::from_dsn(dsn)?; @@ -468,7 +468,7 @@ consumer = Consumer( "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "msg.with.table.name": "true", } ) @@ -488,7 +488,7 @@ let consumer = taos.consumer({ 'group.id': 'tg2', 'td.connect.user': 'root', 'td.connect.pass': 'taosdata', - 'auto.offset.reset','earliest', + 'auto.offset.reset','latest', 'msg.with.table.name': 'true', 'td.connect.ip','127.0.0.1', 'td.connect.port','6030' diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 85bc67d0fd..fd43dd67fa 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -1093,7 +1093,7 @@ TaosConsumer consumer = new TaosConsumer<>(config); - httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type. - messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type. - httpPoolSize: Maximum number of concurrent requests on the a connection。It only takes effect when using WebSocket type. -- For more information, see [Consumer Parameters](../../../develop/tmq). +- For more information, see [Consumer Parameters](../../../develop/tmq). Note that the default value of auto.offset.reset in data subscription on the TDengine server has changed since version 3.2.0.0. #### Subscribe to consume data @@ -1193,7 +1193,7 @@ public abstract class ConsumerLoop { config.setProperty("bootstrap.servers", "localhost:6030"); config.setProperty("td.connect.user", "root"); config.setProperty("td.connect.pass", "taosdata"); - config.setProperty("auto.offset.reset", "earliest"); + config.setProperty("auto.offset.reset", "latest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); config.setProperty("auto.commit.interval.ms", "1000"); @@ -1276,7 +1276,7 @@ public abstract class ConsumerLoop { config.setProperty("bootstrap.servers", "localhost:6041"); config.setProperty("td.connect.user", "root"); config.setProperty("td.connect.pass", "taosdata"); - config.setProperty("auto.offset.reset", "earliest"); + config.setProperty("auto.offset.reset", "latest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); config.setProperty("auto.commit.interval.ms", "1000"); diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java index 3c5d2867e2..8162c30ff6 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -66,7 +66,6 @@ public class SubscribeDemo { properties.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taos.example.MetersDeserializer"); properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8"); - properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true"); // poll data try (TaosConsumer consumer = new TaosConsumer<>(properties)) { diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java index 03f7e3a11e..7df15f1af6 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java @@ -66,7 +66,6 @@ public class WebsocketSubscribeDemo { properties.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taos.example.MetersDeserializer"); properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8"); - properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true"); // poll data try (TaosConsumer consumer = new TaosConsumer<>(properties)) { diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py index 41737e3fc4..c370db47a5 100644 --- a/docs/examples/python/tmq_assignment_example.py +++ b/docs/examples/python/tmq_assignment_example.py @@ -23,9 +23,6 @@ def taos_get_assignment_and_seek_demo(): consumer = Consumer( { "group.id": "0", - # should disable snapshot, - # otherwise it will cause invalid params error - "experimental.snapshot.enable": "false", } ) consumer.subscribe(["tmq_assignment_demo_topic"]) diff --git a/docs/examples/python/tmq_websocket_assgnment_example.py b/docs/examples/python/tmq_websocket_assgnment_example.py index 0f8e4a2804..a180ef840e 100644 --- a/docs/examples/python/tmq_websocket_assgnment_example.py +++ b/docs/examples/python/tmq_websocket_assgnment_example.py @@ -21,9 +21,6 @@ def taosws_get_assignment_and_seek_demo(): prepare() consumer = taosws.Consumer(conf={ "td.connect.websocket.scheme": "ws", - # should disable snapshot, - # otherwise it will cause invalid params error - "experimental.snapshot.enable": "false", "group.id": "0", }) consumer.subscribe(["tmq_assignment_demo_topic"]) diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.mdx similarity index 99% rename from docs/zh/07-develop/07-tmq.md rename to docs/zh/07-develop/07-tmq.mdx index a542c844fe..d3b27fedc3 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.mdx @@ -371,7 +371,7 @@ tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); tmq_conf_set(conf, "group.id", "cgrpName"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); -tmq_conf_set(conf, "auto.offset.reset", "earliest"); +tmq_conf_set(conf, "auto.offset.reset", "latest"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -401,7 +401,7 @@ properties.setProperty("group.id", "cgrpName"); properties.setProperty("bootstrap.servers", "127.0.0.1:6030"); properties.setProperty("td.connect.user", "root"); properties.setProperty("td.connect.pass", "taosdata"); -properties.setProperty("auto.offset.reset", "earliest"); +properties.setProperty("auto.offset.reset", "latest"); properties.setProperty("msg.with.table.name", "true"); properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer"); @@ -441,7 +441,7 @@ consumer, err := NewConsumer(conf) let mut dsn: Dsn = "taos://".parse()?; dsn.set("group.id", "group1"); dsn.set("client.id", "test"); -dsn.set("auto.offset.reset", "earliest"); +dsn.set("auto.offset.reset", "latest"); let tmq = TmqBuilder::from_dsn(dsn)?; @@ -469,7 +469,7 @@ consumer = Consumer( "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "msg.with.table.name": "true", } ) @@ -489,7 +489,7 @@ let consumer = taos.consumer({ 'group.id': 'tg2', 'td.connect.user': 'root', 'td.connect.pass': 'taosdata', - 'auto.offset.reset','earliest', + 'auto.offset.reset','latest', 'msg.with.table.name': 'true', 'td.connect.ip','127.0.0.1', 'td.connect.port','6030' diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 7b6db8ee69..237e3ef8f9 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -1095,7 +1095,8 @@ TaosConsumer consumer = new TaosConsumer<>(config); - httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 - messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 - httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。 - 其他参数请参考：[Consumer 参数列表](../../develop/tmq#创建-consumer-以及consumer-group) + 其他参数请参考：[Consumer 参数列表](../../develop/tmq#创建-consumer-以及consumer-group)， 注意TDengine服务端自3.2.0.0版本开始消息订阅中的auto.offset.reset默认值发生变化。 + #### 订阅消费数据 @@ -1193,7 +1194,7 @@ public abstract class ConsumerLoop { config.setProperty("bootstrap.servers", "localhost:6030"); config.setProperty("td.connect.user", "root"); config.setProperty("td.connect.pass", "taosdata"); - config.setProperty("auto.offset.reset", "earliest"); + config.setProperty("auto.offset.reset", "latest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); config.setProperty("auto.commit.interval.ms", "1000"); @@ -1201,7 +1202,6 @@ public abstract class ConsumerLoop { config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); config.setProperty("value.deserializer.encoding", "UTF-8"); - config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1279,7 +1279,7 @@ public abstract class ConsumerLoop { config.setProperty("bootstrap.servers", "localhost:6041"); config.setProperty("td.connect.user", "root"); config.setProperty("td.connect.pass", "taosdata"); - config.setProperty("auto.offset.reset", "earliest"); + config.setProperty("auto.offset.reset", "latest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); config.setProperty("auto.commit.interval.ms", "1000"); @@ -1287,7 +1287,6 @@ public abstract class ConsumerLoop { config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); config.setProperty("value.deserializer.encoding", "UTF-8"); - config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); From 8fbea08bba7823d8193f27ef14b42c69ed817156 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Oct 2023 09:37:43 +0800 Subject: [PATCH 152/177] enh gen first tag name --- source/dnode/mnode/impl/src/mndStb.c | 2 ++ source/os/src/osRand.c | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index eaf74a96cb..01c9aa5313 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -868,6 +868,8 @@ static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagn int8_t end = left >= 24 ? 24 : left - 1; // gen rand str len [base:end] // note: ignore rand performance issues + taosSeedRand(taosSafeRand()); + int64_t len = taosRand() % (end - start + 1) + start; taosRandStr2(randStr, len); sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index b71be59f1d..0e58b7a8ec 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -86,8 +86,7 @@ void taosRandStr(char* str, int32_t size) { } void taosRandStr2(char* str, int32_t size) { - - const char* set = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ@"; + const char* set = "abcdefghijklmnopqrstuvwxyz0123456789@"; int32_t len = strlen(set); for (int32_t i = 0; i < size; ++i) { From 720982374cc222c205de4c185a6ad6980240be13 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 20 Oct 2023 10:36:52 +0800 Subject: [PATCH 153/177] fix: union all cast func resType check error --- source/libs/parser/src/parAstCreater.c | 4 ++++ source/libs/parser/src/parTranslater.c | 5 +++-- source/libs/parser/src/parUtil.c | 2 +- tests/system-test/2-query/varchar.py | 6 ++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 12062e0d4a..e8614147f4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1379,6 +1379,10 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType if (!checkColumnName(pCxt, pColName) || !checkComment(pCxt, pComment, false)) { return NULL; } + if (IS_VAR_DATA_TYPE(dataType.type) && dataType.bytes == 0) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + return NULL; + } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); CHECK_OUT_OF_MEM(pCol); COPY_STRING_FORM_ID_TOKEN(pCol->colName, pColName); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42ab2f7a2f..756a1f867b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2185,11 +2185,12 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d nodesDestroyNode((SNode*)pFunc); return TSDB_CODE_OUT_OF_MEMORY; } - if (TSDB_CODE_SUCCESS != getFuncInfo(pCxt, pFunc)) { + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS != code) { nodesClearList(pFunc->pParameterList); pFunc->pParameterList = NULL; nodesDestroyNode((SNode*)pFunc); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName); + return code; } *pCast = (SNode*)pFunc; return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1b8bac4cbc..0ab8927bd0 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -126,7 +126,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_FIRST_COLUMN: return "First column must be timestamp"; case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN: - return "Invalid binary/nchar column length"; + return "Invalid column length for var length type"; case TSDB_CODE_PAR_INVALID_TAGS_NUM: return "Invalid number of tag columns"; case TSDB_CODE_PAR_INVALID_INTERNAL_PK: diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index f0849010c6..debcd1f95e 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -76,6 +76,12 @@ class TDTestCase: for i in range(tdSql.queryRows): tdSql.checkData(i,0, data_ct1_c8[i]) + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(0)) tags(tg1 int)") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 nchar(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 binary(0))") + tdSql.error("create stable empty_col_stable(ts timestamp, c2 varchar(10)) tags(tg1 varbinary(0))") + # tdSql.query("select c8 from ct4") # data_ct4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] From c0c6e206d239cf144d793de96a842641005a9d45 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Oct 2023 10:44:33 +0800 Subject: [PATCH 154/177] enh gen first tag name --- source/dnode/mnode/impl/src/mndStb.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 01c9aa5313..8211cb7a96 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -859,21 +859,6 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat return 0; } static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) { - char randStr[TSDB_COL_NAME_LEN] = {0}; - int32_t left = TSDB_COL_NAME_LEN - strlen(tagname) - 1; - if (left <= 1) { - sprintf(fullname, "%s.%s", dbname, tagname); - } else { - int8_t start = left < 8 ? 0 : 8; - int8_t end = left >= 24 ? 24 : left - 1; - // gen rand str len [base:end] - // note: ignore rand performance issues - taosSeedRand(taosSafeRand()); - - int64_t len = taosRand() % (end - start + 1) + start; - taosRandStr2(randStr, len); - sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr); - } return 0; } From f9a027bc7d1824ce2204801efd5af73d27632193 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Oct 2023 10:45:57 +0800 Subject: [PATCH 155/177] enh gen first tag name --- source/dnode/mnode/impl/src/mndStb.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8211cb7a96..bdec03b806 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -858,7 +858,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat } return 0; } -static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) { +static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) { + SName name = {0}; + tNameFromString(&name, stbname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + return snprintf(fullname, TSDB_INDEX_FNAME_LEN - 1, "%s.%s_%s", dbname, tagname, tNameGetTableName(&name)); return 0; } @@ -876,7 +879,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER; SSchema *pSchema = &(stbObj.pTags[0]); - mndGenIdxNameForFirstTag(fullIdxName, pDb->name, pSchema->name); + mndGenIdxNameForFirstTag(fullIdxName, pDb->name, stbObj.name, pSchema->name); SSIdx idx = {0}; if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) { terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; From 6f21d542cde85eb9a94399144790c8ec47984904 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Oct 2023 10:46:17 +0800 Subject: [PATCH 156/177] enh gen first tag name --- source/dnode/mnode/impl/src/mndStb.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index bdec03b806..5516b9d653 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -862,8 +862,6 @@ static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbn SName name = {0}; tNameFromString(&name, stbname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); return snprintf(fullname, TSDB_INDEX_FNAME_LEN - 1, "%s.%s_%s", dbname, tagname, tNameGetTableName(&name)); - - return 0; } static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { From e89d1851c8c4580426691a85ac570289811f4299 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Oct 2023 10:47:02 +0800 Subject: [PATCH 157/177] enh gen first tag name --- source/dnode/mnode/impl/src/mndStb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 5516b9d653..16825d73b1 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -861,7 +861,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) { SName name = {0}; tNameFromString(&name, stbname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - return snprintf(fullname, TSDB_INDEX_FNAME_LEN - 1, "%s.%s_%s", dbname, tagname, tNameGetTableName(&name)); + return snprintf(fullname, TSDB_INDEX_FNAME_LEN, "%s.%s_%s", dbname, tagname, tNameGetTableName(&name)); } static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { From 74d05c2771c4c04b59cb3bf927b2eb6811838e92 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 13:47:25 +0800 Subject: [PATCH 158/177] vnode/cos: fix multipart uploading --- source/dnode/vnode/src/vnd/vnodeCos.c | 71 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index ccda20e049..9c69f2452c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -211,33 +211,6 @@ typedef struct put_object_callback_data { int noStatus; } put_object_callback_data; -static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { - put_object_callback_data *data = (put_object_callback_data *)callbackData; - - int ret = 0; - - if (data->contentLength) { - int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); - if (data->gb) { - growbuffer_read(&(data->gb), toRead, &ret, buffer); - } else if (data->infileFD) { - // ret = fread(buffer, 1, toRead, data->infile); - ret = taosReadFile(data->infileFD, buffer, toRead); - } - } - - data->contentLength -= ret; - data->totalContentLength -= ret; - - if (data->contentLength && !data->noStatus) { - vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); - vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / - data->totalOriginalContentLength)); - } - - return ret; -} - #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { @@ -280,6 +253,37 @@ typedef struct MultipartPartData { UploadManager *manager; } MultipartPartData; +static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { + put_object_callback_data *data = (put_object_callback_data *)callbackData; + if (data->infileFD == 0) { + MultipartPartData *mpd = (MultipartPartData *)callbackData; + data = &mpd->put_object_data; + } + + int ret = 0; + + if (data->contentLength) { + int toRead = ((data->contentLength > (unsigned)bufferSize) ? (unsigned)bufferSize : data->contentLength); + if (data->gb) { + growbuffer_read(&(data->gb), toRead, &ret, buffer); + } else if (data->infileFD) { + // ret = fread(buffer, 1, toRead, data->infile); + ret = taosReadFile(data->infileFD, buffer, toRead); + } + } + + data->contentLength -= ret; + data->totalContentLength -= ret; + /* log too many open files + if (data->contentLength && !data->noStatus) { + vTrace("%llu bytes remaining ", (unsigned long long)data->totalContentLength); + vTrace("(%d%% complete) ...\n", (int)(((data->totalOriginalContentLength - data->totalContentLength) * 100) / + data->totalOriginalContentLength)); + } + */ + return ret; +} + S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { UploadManager *manager = (UploadManager *)callbackData; manager->upload_id = strdup(upload_id); @@ -308,7 +312,7 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackD manager->remaining -= ret; return ret; } - +/* static S3Status listPartsCallback(int isTruncated, const char *nextPartNumberMarker, const char *initiatorId, const char *initiatorDisplayName, const char *ownerId, const char *ownerDisplayName, const char *storageClass, int partsCount, int handlePartsStart, @@ -418,7 +422,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan return 0; } - +*/ int32_t s3PutObjectFromFile2(const char *file, const char *object) { int32_t code = 0; const char *key = object; @@ -480,9 +484,11 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { if (data.status != S3StatusOK) { s3PrintError(__func__, data.status, data.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data.contentLength); + code = TAOS_SYSTEM_ERROR(EIO); } } else { uint64_t totalContentLength = contentLength; @@ -493,7 +499,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // div round up int seq; + // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); + // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -530,6 +538,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { if (manager.upload_id == 0 || manager.status != S3StatusOK) { s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -554,6 +563,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(partData.status) && should_retry()); if (partData.status != S3StatusOK) { s3PrintError(__func__, partData.status, partData.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } contentLength -= MULTIPART_CHUNK_SIZE; @@ -581,6 +591,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { s3PrintError(__func__, manager.status, manager.err_msg); + code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -595,7 +606,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { taosMemoryFree(manager.etags); } - return 0; + return code; } typedef struct list_bucket_callback_data { From f016dab40af2446310ecfa58871405af880b11ea Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 13:59:01 +0800 Subject: [PATCH 159/177] common/global: comment s3BlockSize altering out --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f3af6e0ab0..9bc981b520 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1683,14 +1683,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) { } return; } - + /* cannot alter s3BlockSize if (strcasecmp(option, "s3BlockSize") == 0) { int32_t newS3BlockSize = atoi(value); uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize); tsS3BlockSize = newS3BlockSize; return; } - + */ if (strcasecmp(option, "s3BlockCacheSize") == 0) { int32_t newS3BlockCacheSize = atoi(value); uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize); From 55647f9ce9188e79405345935838bcd009f6f898 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Oct 2023 15:15:03 +0800 Subject: [PATCH 160/177] cos/put: make chunk small to not block other reqs --- source/dnode/vnode/src/vnd/vnodeCos.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 9c69f2452c..6e36739f5a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -498,10 +498,9 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { manager.gb = 0; // div round up - int seq; - // uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; - int totalSeq = ((contentLength + MULTIPART_CHUNK_SIZE - 1) / MULTIPART_CHUNK_SIZE); - // int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); + int seq; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; + int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -543,14 +542,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } upload: - todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; + todoContentLength -= chunk_size * manager.next_etags_pos; for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { partData.manager = &manager; partData.seq = seq; if (partData.put_object_data.gb == NULL) { partData.put_object_data = data; } - partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); + partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength); // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); partData.put_object_data.contentLength = partContentLength; partData.put_object_data.originalContentLength = partContentLength; @@ -566,8 +565,8 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { code = TAOS_SYSTEM_ERROR(EIO); goto clean; } - contentLength -= MULTIPART_CHUNK_SIZE; - todoContentLength -= MULTIPART_CHUNK_SIZE; + contentLength -= chunk_size; + todoContentLength -= chunk_size; } int i; From f667bf3866fcda26041044d594e4b75df65c451d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 20 Oct 2023 18:16:19 +0800 Subject: [PATCH 161/177] fix(tsdb): apply the window range when building blocks from stt files. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 28 +++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index db1de7742c..80f4d64cd5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2670,6 +2670,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); @@ -2705,8 +2706,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } else { bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN; - if (!bHasDataInLastBlock || ((ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.lastKey < tsLast) || - (!ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.firstKey > tsLast))) { + if (!bHasDataInLastBlock || + ((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; pInfo->rows = pBlockInfo->record.numRow; @@ -2728,26 +2729,28 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { tBlockDataReset(pBData); SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - tsdbDebug("load data in last block firstly %s", pReader->idStr); + tsdbDebug("load data in last block firstly %s", pReader->idStr); int64_t st = taosGetTimestampUs(); - while (1) { - bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); - - // no data in last block and block, no need to proceed. - if (hasBlockLData == false) { - break; - } - + // no data in last block, no need to proceed. + while (hasDataInLastBlock(pLastBlockReader)) { code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); - if (code) { + if (code != TSDB_CODE_SUCCESS) { return code; } if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { break; } + + // data in stt now overlaps with current active file data block, need to composed with file data block. + int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader); + if ((keyInStt >= pBlockInfo->record.firstKey && asc) || (keyInStt <= pBlockInfo->record.lastKey && (!asc))) { + tsdbDebug("%p keyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, + keyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + break; + } } double el = (taosGetTimestampUs() - st) / 1000.0; @@ -2760,7 +2763,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pResBlock->info.rows, el, pReader->idStr); } } - } return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; From f630fc38f805e178ef1c3e0b27b68a16036f2d80 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 15:46:32 +0800 Subject: [PATCH 162/177] tsdb/commit: skip last tier ts data committing --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index d4cb63fb7b..5bb486de4a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -128,8 +128,16 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { } } + extern int8_t tsS3Enabled; + + int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); + bool skipRow = false; + if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) { + skipRow = true; + } + int64_t ts = TSDBROW_TS(&row->row); - if (ts > committer->ctx->maxKey) { + if (ts > committer->ctx->maxKey || skipRow) { committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); From f8097979995e3620a44dd8106a7cbf540a4e4d08 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 23 Oct 2023 16:04:45 +0800 Subject: [PATCH 163/177] enh: taosd build info --- cmake/cmake.version | 14 +++++++++++++- source/util/src/version.c.in | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index fa6ec4df17..c0ddae569c 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -50,7 +50,19 @@ ENDIF () IF (DEFINED VERDATE) SET(TD_VER_DATE ${VERDATE}) ELSE () - STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + STRING(COMPARE GREATER_EQUAL "${CMAKE_VERSION}" "3.26" TD_CMAKE_SUPPORT_TZ) + + IF (TD_CMAKE_SUPPORT_TZ) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S %z") + ELSE () + IF (TD_WINDOWS) + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + ELSE () + EXECUTE_PROCESS(COMMAND date +"%F %T %z" OUTPUT_VARIABLE TD_VER_DATE) + STRING(REPLACE "\"" "" TD_VER_DATE ${TD_VER_DATE}) + STRING(STRIP ${TD_VER_DATE} TD_VER_DATE) + ENDIF () + ENDIF () ENDIF () IF (DEFINED VERTYPE) diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index 71998e3321..ec6449a02f 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -2,6 +2,6 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; -char buildinfo[64] = "Built at ${TD_VER_DATE}"; +char buildinfo[64] = "Built ${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} at ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From 62cd6f3ed26d1aa8b8e064de24e651b1b2d1d0eb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 16:50:51 +0800 Subject: [PATCH 164/177] cmake/curl, libs3, ssl: clean source dir before building --- cmake/curl_CMakeLists.txt.in | 7 ++++--- cmake/libs3_CMakeLists.txt.in | 4 ++-- cmake/ssl_CMakeLists.txt.in | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 3f7b9ee1fe..2fe84b9721 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -12,9 +12,10 @@ ExternalProject_Add(curl BUILD_ALWAYS 1 #UPDATE_COMMAND "" #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug + #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND make - INSTALL_COMMAND make install + BUILD_COMMAND "" + INSTALL_COMMAND make clean && make install TEST_COMMAND "" ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b2d2d19059..6c911775ee 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -23,7 +23,7 @@ ExternalProject_Add(libs3 CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a - BUILD_COMMAND make build/lib/libs3.a - INSTALL_COMMAND make install_static + BUILD_COMMAND "" + INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static TEST_COMMAND "" ) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 65a02d7fd6..8301dbe222 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -10,6 +10,6 @@ ExternalProject_Add(openssl #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared BUILD_COMMAND "" - INSTALL_COMMAND make install_sw -j + INSTALL_COMMAND make clean && make install_sw -j TEST_COMMAND "" ) From b864c87b7c91d8663c592f25ee681f85cc3e761f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 17:00:52 +0800 Subject: [PATCH 165/177] cmake/ssl: make depend --- cmake/ssl_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 8301dbe222..c17043ff22 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -10,6 +10,6 @@ ExternalProject_Add(openssl #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared BUILD_COMMAND "" - INSTALL_COMMAND make clean && make install_sw -j + INSTALL_COMMAND make clean && make depend && make install_sw -j TEST_COMMAND "" ) From 7cf72a567da657f1c2f62401722dada5e757283b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 19:19:34 +0800 Subject: [PATCH 166/177] cmake/ssl: use build command --- cmake/ssl_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index c17043ff22..1f3fd0a30d 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -9,7 +9,7 @@ ExternalProject_Add(openssl #BUILD_ALWAYS 1 #UPDATE_COMMAND "" CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make depend && make install_sw -j + BUILD_COMMAND make -j + INSTALL_COMMAND make install_sw -j TEST_COMMAND "" ) From 7d17c4cc90385350805fb90a3501796486dd48e6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 23 Oct 2023 19:35:22 +0800 Subject: [PATCH 167/177] cmake/curl, libs3: use build command --- cmake/curl_CMakeLists.txt.in | 4 ++-- cmake/libs3_CMakeLists.txt.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 2fe84b9721..b013ed1807 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -15,7 +15,7 @@ ExternalProject_Add(curl CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug #CONFIGURE_COMMAND ./configure --without-ssl - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make install + BUILD_COMMAND make -j + INSTALL_COMMAND make install TEST_COMMAND "" ) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 6c911775ee..b2d2d19059 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -23,7 +23,7 @@ ExternalProject_Add(libs3 CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a - BUILD_COMMAND "" - INSTALL_COMMAND make clean && make build/lib/libs3.a && make install_static + BUILD_COMMAND make build/lib/libs3.a + INSTALL_COMMAND make install_static TEST_COMMAND "" ) From 7fdf6cc696fae234f993479f5737451a384cfda3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 09:04:24 +0800 Subject: [PATCH 168/177] cmake: use new dir for libs3 --- cmake/curl_CMakeLists.txt.in | 7 ++----- cmake/libs3.GNUmakefile | 2 +- cmake/libs3_CMakeLists.txt.in | 15 +-------------- cmake/ssl_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 4 ++-- source/dnode/vnode/CMakeLists.txt | 8 ++++---- tests/parallel_test/container_build.sh | 4 ++-- 7 files changed, 13 insertions(+), 29 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index b013ed1807..140fc48103 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -10,11 +10,8 @@ ExternalProject_Add(curl DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 - #UPDATE_COMMAND "" - #CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --with-ssl=$ENV{HOME}/.cos-local.1 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug - - #CONFIGURE_COMMAND ./configure --without-ssl + UPDATE_COMMAND "" + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" diff --git a/cmake/libs3.GNUmakefile b/cmake/libs3.GNUmakefile index abf954f5c1..98f98d8224 100644 --- a/cmake/libs3.GNUmakefile +++ b/cmake/libs3.GNUmakefile @@ -91,7 +91,7 @@ endif # -------------------------------------------------------------------------- # DESTDIR directory ifndef DESTDIR - DESTDIR := ${HOME}/.cos-local.1 + DESTDIR := ${HOME}/.cos-local.2 endif # -------------------------------------------------------------------------- diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b2d2d19059..b5e9c438e6 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -1,15 +1,5 @@ # libs3 -set(s3_flags "${CMAKE_C_FLAGS} -O3 -Iinc -I$ENV{HOME}/.cos-local.1/include -Iinc ") - -set(SED_CMD "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /") - -function(update_cflags) - file(READ "GNUmakefile" S3_CONTENT) - string(REPLACE "CFLAGS += -Wall -Werror" "CFLAGS += -I$ENV{HOME}/.cos-local.1/include " S3_CONTENT "${S3_CONTENT}") - file(WRITE "GNUmakefile" "${S3_CONTENT}") -endfunction(update_cflags) - ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 @@ -19,10 +9,7 @@ ExternalProject_Add(libs3 BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 UPDATE_COMMAND "" - #sed_cmd "s/CFLAGS += -Wall -Werror/CFLAGS += -I$ENV{HOME}/.cos-local.1/include /" - CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.1/include' -L'$ENV{HOME}/.cos-local.1/lib' |" ./GNUmakefile - #BUILD_COMMAND make CFLAGS=${s3_flags} DESTDIR=$ENV{HOME}/.cos-local.1 build/lib/libs3.a - #BUILD_COMMAND make DESTDIR="$ENV{HOME}/.cos-local.1" build/lib/libs3.a + CONFIGURE_COMMAND cp ${TD_SUPPORT_DIR}/libs3.GNUmakefile GNUmakefile && sed -i "s|CFLAGS += -Wall -Werror|CFLAGS += -I'$ENV{HOME}/.cos-local.2/include' -L'$ENV{HOME}/.cos-local.2/lib' |" ./GNUmakefile BUILD_COMMAND make build/lib/libs3.a INSTALL_COMMAND make install_static TEST_COMMAND "" diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index 1f3fd0a30d..c9f836bade 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -8,7 +8,7 @@ ExternalProject_Add(openssl BUILD_IN_SOURCE TRUE #BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.1 -static #--no-shared + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared BUILD_COMMAND make -j INSTALL_COMMAND make install_sw -j TEST_COMMAND "" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 013269c583..e2a5378f0d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -9,7 +9,7 @@ endfunction(cat IN_FILE OUT_FILE) if(${TD_LINUX}) if(${BUILD_WITH_S3}) - file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) + file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) else() @@ -414,7 +414,7 @@ if (${BUILD_WITH_ROCKSDB}) endif() if(${BUILD_WITH_S3}) - INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") else() diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 7e2693e8af..bc93ab575e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -167,15 +167,15 @@ if(${BUILD_WITH_S3}) target_include_directories( vnode - PUBLIC "$ENV{HOME}/.cos-local.1/include" + PUBLIC "$ENV{HOME}/.cos-local.2/include" ) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl) - find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.1/lib64) - find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.1/lib64) + find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64) + find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64) target_link_libraries( vnode diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index f5e426057e..94704b1c25 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -60,7 +60,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ @@ -89,7 +89,7 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.1 \ + -v /root/.cos-local.1:/root/.cos-local.2 \ -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ From fd87f43f8cdfbb3668bfe011b14d1ce2d3794567 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 09:10:58 +0800 Subject: [PATCH 169/177] enh: set CPUTYPE in cmake --- cmake/cmake.platform | 6 ++++++ cmake/cmake.version | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 18fd17f018..ca2cbbec2e 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -96,33 +96,39 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(CPUTYPE "x64") SET(TD_INTEL_64 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(CPUTYPE "x86") SET(TD_INTEL_32 TRUE) ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(CPUTYPE "arm32") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "Current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(CPUTYPE "arm64") SET(TD_ARM_64 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") MESSAGE(STATUS "The current platform is loongarch64") SET(PLATFORM_ARCH_STR "loongarch64") + SET(CPUTYPE "loongarch64") SET(TD_LOONGARCH_64 TRUE) ADD_DEFINITIONS("-D_TD_LOONGARCH_") ADD_DEFINITIONS("-D_TD_LOONGARCH_64") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") + SET(CPUTYPE "mips64") MESSAGE(STATUS "input cpuType: mips64") SET(TD_MIPS_64 TRUE) ADD_DEFINITIONS("-D_TD_MIPS_") diff --git a/cmake/cmake.version b/cmake/cmake.version index c0ddae569c..b8871dbce3 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -79,9 +79,9 @@ ELSE () ELSEIF (TD_LINUX_32) SET(TD_VER_CPUTYPE "x86") ELSEIF (TD_ARM_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "arm32") ELSEIF (TD_MIPS_32) - SET(TD_VER_CPUTYPE "x86") + SET(TD_VER_CPUTYPE "mips32") ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () From 0e65ed0202bb772908a8bc6867e6d1d2078c4f91 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 09:16:09 +0800 Subject: [PATCH 170/177] cmake/curl: use new dir curl2 for s3 version --- cmake/curl_CMakeLists.txt.in | 4 ++-- cmake/libs3_CMakeLists.txt.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 140fc48103..458a518092 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,12 +1,12 @@ # curl -ExternalProject_Add(curl +ExternalProject_Add(curl2 URL https://curl.se/download/curl-8.2.1.tar.gz URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" #GIT_REPOSITORY https://github.com/curl/curl.git #GIT_TAG curl-7_88_1 - SOURCE_DIR "${TD_CONTRIB_DIR}/curl" + SOURCE_DIR "${TD_CONTRIB_DIR}/curl2" DEPENDS openssl BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index b5e9c438e6..f2b6cac953 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -3,7 +3,7 @@ ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl2 xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE From 37131564a3c3c86df52bd47c4500b5631ff100e3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 24 Oct 2023 09:54:52 +0800 Subject: [PATCH 171/177] fix:rollback processing with # --- tests/pytest/util/dnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 5f1522bfd9..c4fc1ce654 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -64,7 +64,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo %s '%s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) From e826532592d520de945ca6fcf771123ae967e8fa Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 10:10:43 +0800 Subject: [PATCH 172/177] cmake/vnode: specify no default path for ssl & crypto --- source/dnode/vnode/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bc93ab575e..b83d64c569 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -174,8 +174,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl) - find_library(SSL_LIBRARY ssl PATHS $ENV{HOME}/.cos-local.2/lib64) - find_library(CRYPTO_LIBRARY crypto PATHS $ENV{HOME}/.cos-local.2/lib64) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( vnode From a3fd13d43aa1fe856fc50c53aca2c89e972863a0 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 24 Oct 2023 10:14:12 +0800 Subject: [PATCH 173/177] Update test_R.sh --- tests/docs-examples-test/test_R.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs-examples-test/test_R.sh b/tests/docs-examples-test/test_R.sh index 707ea02704..d56f8973f7 100755 --- a/tests/docs-examples-test/test_R.sh +++ b/tests/docs-examples-test/test_R.sh @@ -6,7 +6,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 & cd ../../docs/examples/R -wget -N https://repo1.maven.org/maven2/com/taosdata/jdbc/taos-jdbcdriver/3.2.4/taos-jdbcdriver-3.2.4-dist.jar +wget -N https://maven.aliyun.com/repository/central/com/taosdata/jdbc/taos-jdbcdriver/3.2.5/taos-jdbcdriver-3.2.5-dist.jar jar_path=`find . -name taos-jdbcdriver-*-dist.jar` echo jar_path=$jar_path From be7791f9cd25b1362907b414773ce6b5daf4ff33 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 24 Oct 2023 13:58:30 +0800 Subject: [PATCH 174/177] fix: disable life extending of conn cache obj when show queries/conns --- include/util/tcache.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 13 +++++++------ source/util/src/tcache.c | 12 ++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/include/util/tcache.h b/include/util/tcache.h index d8ab018570..474e5274de 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -151,6 +151,8 @@ void *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen); void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen); void taosCacheDestroyIter(SCacheIter *pIter); +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..4366053237 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -59,7 +59,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId); -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq); @@ -79,7 +79,7 @@ int32_t mndInitProfile(SMnode *pMnode) { // in ms int32_t checkTime = tsShellActivityTimer * 2 * 1000; - pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc profile cache since %s", terrstr()); @@ -185,11 +185,12 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) { return pConn; } -static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn, bool extendLifespan) { if (pConn == NULL) return; mTrace("conn:%u, released from cache, data:%p", pConn->id, pConn); SProfileMgmt *pMgmt = &pMnode->profileMgmt; + if (extendLifespan) taosCacheTryExtendLifeSpan(pMgmt->connCache, (void **)&pConn); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); } @@ -323,7 +324,7 @@ _OVER: mndReleaseUser(pMnode, pUser); mndReleaseDb(pMnode, pDb); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); return code; } @@ -485,7 +486,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr()); return -1; @@ -508,7 +509,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn, true); hbRsp.query = rspBasic; } else { diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 392ac5d8b2..11f8df4c93 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -997,3 +997,15 @@ void taosCacheDestroyIter(SCacheIter *pIter) { taosMemoryFreeClear(pIter->pCurrent); taosMemoryFreeClear(pIter); } + +void taosCacheTryExtendLifeSpan(SCacheObj *pCacheObj, void **data) { + if (!pCacheObj || !(*data)) return; + + SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); + if (pNode->signature != pNode) return; + + if (!pNode->inTrashcan) { + atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); + uDebug("cache:%s, data:%p extend expire time: %" PRId64, pCacheObj->name, pNode->data, pNode->expireTime); + } +} From f9ca8fe9620e57f0c64c3ebb62d4dca2647352d9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 16:10:46 +0800 Subject: [PATCH 175/177] cmake: new option: BUILD_S3 --- CMakeLists.txt | 4 +--- cmake/cmake.options | 28 +++++++++++++++++++++------- contrib/CMakeLists.txt | 23 +++++++---------------- source/common/CMakeLists.txt | 14 +++++--------- source/dnode/vnode/CMakeLists.txt | 6 +++--- 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5048287cf6..66a6fd328d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,6 @@ set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") - - include(${TD_SUPPORT_DIR}/cmake.platform) include(${TD_SUPPORT_DIR}/cmake.define) include(${TD_SUPPORT_DIR}/cmake.options) @@ -46,4 +44,4 @@ add_subdirectory(examples/c) include(${TD_SUPPORT_DIR}/cmake.install) # docs -add_subdirectory(docs/doxgen) \ No newline at end of file +add_subdirectory(docs/doxgen) diff --git a/cmake/cmake.options b/cmake/cmake.options index bacc5adfb7..d34c34dd89 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -128,12 +128,16 @@ option( IF(${TD_LINUX}) option( - BUILD_WITH_S3 + BUILD_S3 "If build with s3" ON ) -IF(${BUILD_WITH_S3}) +option( + BUILD_WITH_S3 + "If build with s3" + ON +) option( BUILD_WITH_COS @@ -141,16 +145,26 @@ option( OFF ) +ENDIF () + +IF(${BUILD_S3}) + +IF(${BUILD_WITH_S3}) + +option(BUILD_WITH_COS "If build with cos" OFF) + ELSE () -option( - BUILD_WITH_COS - "If build with cos" - ON -) +option(BUILD_WITH_COS "If build with cos" ON) ENDIF () +ELSE () + +option(BUILD_WITH_S3 "If build with s3" OFF) + +option(BUILD_WITH_COS "If build with cos" OFF) + ENDIF () option( diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e2a5378f0d..d631ceff11 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -6,12 +6,10 @@ function(cat IN_FILE OUT_FILE) file(APPEND ${OUT_FILE} "${CONTENTS}") endfunction(cat IN_FILE OUT_FILE) -if(${TD_LINUX}) - if(${BUILD_WITH_S3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.2/) -else() +elseif(${BUILD_WITH_COS}) set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) @@ -42,9 +40,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") -endif(${BUILD_WITH_S3}) +endif() -endif(${TD_LINUX}) set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -170,19 +167,16 @@ if(${BUILD_WITH_S3}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) #cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_COS) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene if(${BUILD_WITH_LUCENE}) @@ -269,7 +263,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 if(${BUILD_WITH_S3}) add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif(${BUILD_WITH_S3}) +endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) @@ -417,10 +411,8 @@ if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") -else() - # cos -if(${BUILD_WITH_COS}) +elseif(${BUILD_WITH_COS}) if(${TD_LINUX}) set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1) #ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a) @@ -442,9 +434,8 @@ if(${BUILD_WITH_COS}) else() endif(${TD_LINUX}) -endif(${BUILD_WITH_COS}) -endif(${BUILD_WITH_S3}) +endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 6bdcdf7d1d..c35845f9df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -17,15 +17,11 @@ IF (TD_STORAGE) ADD_DEFINITIONS(-D_STORAGE) TARGET_LINK_LIBRARIES(common PRIVATE storage) - IF(${TD_LINUX}) - IF(${BUILD_WITH_COS}) - add_definitions(-DUSE_COS) - ENDIF(${BUILD_WITH_COS}) - - IF(${BUILD_WITH_S3}) - add_definitions(-DUSE_S3) - ENDIF(${BUILD_WITH_S3}) - ENDIF(${TD_LINUX}) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ELSEIF(${BUILD_WITH_COS}) + add_definitions(-DUSE_COS) + ENDIF() ENDIF () diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b83d64c569..a10b38a1bf 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -161,7 +161,7 @@ target_link_libraries( PUBLIC index ) -if(${TD_LINUX}) +if(${BUILD_S3}) if(${BUILD_WITH_S3}) target_include_directories( @@ -188,7 +188,7 @@ if(${BUILD_WITH_S3}) ) add_definitions(-DUSE_S3) -endif(${BUILD_WITH_S3}) +endif() if(${BUILD_WITH_COS}) @@ -227,7 +227,7 @@ target_include_directories( add_definitions(-DUSE_COS) endif(${BUILD_WITH_COS}) -endif(${TD_LINUX}) +endif() IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) From aa7fd64b9b964781493da67985b591f713605be4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 24 Oct 2023 18:04:50 +0800 Subject: [PATCH 176/177] enh: cmake system process support x86_64 --- cmake/cmake.platform | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index ca2cbbec2e..9b96ebe7cb 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -93,7 +93,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") ENDIF() IF ("${CPUTYPE}" STREQUAL "") - IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)|(x86_64)|(X86_64)") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(CPUTYPE "x64") From c67853888f80030a29878149e608e20324619903 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 24 Oct 2023 18:06:19 +0800 Subject: [PATCH 177/177] cmake/xml2: build xml2 before s3 --- cmake/xml2_CMakeLists.txt.in | 16 ++++++++++------ contrib/CMakeLists.txt | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 4c234d7a37..25eb5a69ac 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,12 +1,16 @@ # xml2 ExternalProject_Add(xml2 - GIT_REPOSITORY https://github.com/GNOME/libxml2 - GIT_TAG v2.11.5 + URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + #GIT_REPOSITORY https://github.com/GNOME/libxml2 + #GIT_TAG v2.11.5 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" + #BINARY_DIR "" + BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + BUILD_COMMAND make -j + INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d631ceff11..c5715bd53f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -261,9 +261,9 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # xml2 -if(${BUILD_WITH_S3}) - add_subdirectory(xml2 EXCLUDE_FROM_ALL) -endif() +#if(${BUILD_WITH_S3}) +# add_subdirectory(xml2 EXCLUDE_FROM_ALL) +#endif() # lz4 add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)