From 1c27c2123fcb2e88114709c0e55bfaab5084c9d9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 20 Sep 2023 11:06:59 +0800 Subject: [PATCH 01/47] enhance: select `tbname` from (select tbname from d.st) --- include/libs/nodes/querynodes.h | 1 + source/libs/parser/inc/parAst.h | 1 + source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/parAstCreater.c | 15 ++ source/libs/parser/src/parTranslater.c | 3 +- source/libs/parser/src/sql.c | 233 +++++++++++++------------ 6 files changed, 138 insertions(+), 117 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 972421b1e7..58cc3c4a75 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -35,6 +35,7 @@ typedef struct SRawExprNode { char* p; uint32_t n; SNode* pNode; + bool isPseudoColumn; } SRawExprNode; typedef struct SDataType { diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 47043fddb1..3024eea397 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -98,6 +98,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode); SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode); +SNode* setRawExprNodeIsPseudoColumn(SAstCreateContext* pCxt, SNode* pNode, bool isPseudoColumn); SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode); SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 667b21893e..c7da1e0878 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -809,7 +809,7 @@ expr_or_subquery(A) ::= expression(B). //expr_or_subquery(A) ::= subquery(B). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), NULL); } expression(A) ::= literal(B). { A = B; } -expression(A) ::= pseudo_column(B). { A = B; } +expression(A) ::= pseudo_column(B). { A = B; setRawExprNodeIsPseudoColumn(pCxt, A, true); } expression(A) ::= column_reference(B). { A = B; } expression(A) ::= function_expression(B). { A = B; } expression(A) ::= case_when_expression(B). { A = B; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index e0b135b4f9..9baacf2790 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -257,6 +257,15 @@ SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const return (SNode*)target; } +SNode* setRawExprNodeIsPseudoColumn(SAstCreateContext* pCxt, SNode* pNode, bool isPseudoColumn) { + CHECK_PARSER_STATUS(pCxt); + if (NULL == pNode || QUERY_NODE_RAW_EXPR != nodeType(pNode)) { + return pNode; + } + ((SRawExprNode*)pNode)->isPseudoColumn = isPseudoColumn; + return pNode; +} + SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { CHECK_PARSER_STATUS(pCxt); SRawExprNode* pRawExpr = (SRawExprNode*)pNode; @@ -266,6 +275,12 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { if (QUERY_NODE_COLUMN == nodeType(pExpr)) { strcpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName); strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); + } else if (pRawExpr->isPseudoColumn) { + int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); + strncpy(pExpr->userAlias, pRawExpr->p, len); + pExpr->userAlias[len] = '\0'; + strncpy(pExpr->aliasName, pRawExpr->p, len); + pExpr->aliasName[len] = '\0'; } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c702400526..0fc57d3729 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1576,8 +1576,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN return TSDB_CODE_SUCCESS; } if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable || - QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } else { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ff1d834791..b86edc8791 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -6814,7 +6814,6 @@ static YYACTIONTYPE yy_reduce( case 405: /* signed_literal ::= signed */ yytestcase(yyruleno==405); case 426: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==426); case 427: /* expression ::= literal */ yytestcase(yyruleno==427); - case 428: /* expression ::= pseudo_column */ yytestcase(yyruleno==428); case 429: /* expression ::= column_reference */ yytestcase(yyruleno==429); case 430: /* expression ::= function_expression */ yytestcase(yyruleno==430); case 431: /* expression ::= case_when_expression */ yytestcase(yyruleno==431); @@ -6833,37 +6832,37 @@ static YYACTIONTYPE yy_reduce( case 597: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==597); #line 727 "sql.y" { yylhsminor.yy122 = yymsp[0].minor.yy122; } -#line 6836 "sql.c" +#line 6835 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 396: /* literal ::= NULL */ #line 728 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } -#line 6842 "sql.c" +#line 6841 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 397: /* literal ::= NK_QUESTION */ #line 729 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6848 "sql.c" +#line 6847 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 398: /* duration_literal ::= NK_VARIABLE */ #line 731 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6854 "sql.c" +#line 6853 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 399: /* signed ::= NK_INTEGER */ #line 733 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6860 "sql.c" +#line 6859 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 400: /* signed ::= NK_PLUS NK_INTEGER */ #line 734 "sql.y" { yymsp[-1].minor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6866 "sql.c" +#line 6865 "sql.c" break; case 401: /* signed ::= NK_MINUS NK_INTEGER */ #line 735 "sql.y" @@ -6872,19 +6871,19 @@ static YYACTIONTYPE yy_reduce( t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -#line 6875 "sql.c" +#line 6874 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 402: /* signed ::= NK_FLOAT */ #line 740 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6881 "sql.c" +#line 6880 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 403: /* signed ::= NK_PLUS NK_FLOAT */ #line 741 "sql.y" { yymsp[-1].minor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6887 "sql.c" +#line 6886 "sql.c" break; case 404: /* signed ::= NK_MINUS NK_FLOAT */ #line 742 "sql.y" @@ -6893,25 +6892,25 @@ static YYACTIONTYPE yy_reduce( t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } -#line 6896 "sql.c" +#line 6895 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 406: /* signed_literal ::= NK_STRING */ #line 749 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } -#line 6902 "sql.c" +#line 6901 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 407: /* signed_literal ::= NK_BOOL */ #line 750 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } -#line 6908 "sql.c" +#line 6907 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 408: /* signed_literal ::= TIMESTAMP NK_STRING */ #line 751 "sql.y" { yymsp[-1].minor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 6914 "sql.c" +#line 6913 "sql.c" break; case 409: /* signed_literal ::= duration_literal */ case 411: /* signed_literal ::= literal_func */ yytestcase(yyruleno==411); @@ -6923,19 +6922,25 @@ static YYACTIONTYPE yy_reduce( case 611: /* search_condition ::= common_expression */ yytestcase(yyruleno==611); #line 752 "sql.y" { yylhsminor.yy122 = releaseRawExprNode(pCxt, yymsp[0].minor.yy122); } -#line 6926 "sql.c" +#line 6925 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 410: /* signed_literal ::= NULL */ #line 753 "sql.y" { yylhsminor.yy122 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } -#line 6932 "sql.c" +#line 6931 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 412: /* signed_literal ::= NK_QUESTION */ #line 755 "sql.y" { yylhsminor.yy122 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } -#line 6938 "sql.c" +#line 6937 "sql.c" + yymsp[0].minor.yy122 = yylhsminor.yy122; + break; + case 428: /* expression ::= pseudo_column */ +#line 812 "sql.y" +{ yylhsminor.yy122 = yymsp[0].minor.yy122; setRawExprNodeIsPseudoColumn(pCxt, yylhsminor.yy122, true); } +#line 6943 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 432: /* expression ::= NK_LP expression NK_RP */ @@ -6943,7 +6948,7 @@ static YYACTIONTYPE yy_reduce( case 610: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==610); #line 816 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 6946 "sql.c" +#line 6951 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 433: /* expression ::= NK_PLUS expr_or_subquery */ @@ -6952,7 +6957,7 @@ static YYACTIONTYPE yy_reduce( SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy122)); } -#line 6955 "sql.c" +#line 6960 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 434: /* expression ::= NK_MINUS expr_or_subquery */ @@ -6961,7 +6966,7 @@ static YYACTIONTYPE yy_reduce( SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy122), NULL)); } -#line 6964 "sql.c" +#line 6969 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 435: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ @@ -6971,7 +6976,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 6974 "sql.c" +#line 6979 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 436: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ @@ -6981,7 +6986,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 6984 "sql.c" +#line 6989 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 437: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ @@ -6991,7 +6996,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 6994 "sql.c" +#line 6999 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 438: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ @@ -7001,7 +7006,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7004 "sql.c" +#line 7009 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 439: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ @@ -7011,7 +7016,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7014 "sql.c" +#line 7019 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 440: /* expression ::= column_reference NK_ARROW NK_STRING */ @@ -7020,7 +7025,7 @@ static YYACTIONTYPE yy_reduce( SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } -#line 7023 "sql.c" +#line 7028 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 441: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ @@ -7030,7 +7035,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7033 "sql.c" +#line 7038 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 442: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ @@ -7040,19 +7045,19 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7043 "sql.c" +#line 7048 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 445: /* column_reference ::= column_name */ #line 870 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy203, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy203)); } -#line 7049 "sql.c" +#line 7054 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 446: /* column_reference ::= table_name NK_DOT column_name */ #line 871 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy203, createColumnNode(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy203)); } -#line 7055 "sql.c" +#line 7060 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 447: /* pseudo_column ::= ROWTS */ @@ -7069,68 +7074,68 @@ static YYACTIONTYPE yy_reduce( case 464: /* literal_func ::= NOW */ yytestcase(yyruleno==464); #line 873 "sql.y" { yylhsminor.yy122 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } -#line 7072 "sql.c" +#line 7077 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 449: /* pseudo_column ::= table_name NK_DOT TBNAME */ #line 875 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy203)))); } -#line 7078 "sql.c" +#line 7083 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 459: /* function_expression ::= function_name NK_LP expression_list NK_RP */ case 460: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==460); #line 886 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy203, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy203, yymsp[-1].minor.yy298)); } -#line 7085 "sql.c" +#line 7090 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 461: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ #line 889 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), yymsp[-1].minor.yy388)); } -#line 7091 "sql.c" +#line 7096 "sql.c" yymsp[-5].minor.yy122 = yylhsminor.yy122; break; case 463: /* literal_func ::= noarg_func NK_LP NK_RP */ #line 892 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy203, NULL)); } -#line 7097 "sql.c" +#line 7102 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 478: /* star_func_para_list ::= NK_STAR */ #line 916 "sql.y" { yylhsminor.yy298 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } -#line 7103 "sql.c" +#line 7108 "sql.c" yymsp[0].minor.yy298 = yylhsminor.yy298; break; case 483: /* star_func_para ::= table_name NK_DOT NK_STAR */ case 551: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==551); #line 925 "sql.y" { yylhsminor.yy122 = createColumnNode(pCxt, &yymsp[-2].minor.yy203, &yymsp[0].minor.yy0); } -#line 7110 "sql.c" +#line 7115 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 484: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ #line 928 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy298, yymsp[-1].minor.yy122)); } -#line 7116 "sql.c" +#line 7121 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 485: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ #line 930 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), yymsp[-2].minor.yy298, yymsp[-1].minor.yy122)); } -#line 7122 "sql.c" +#line 7127 "sql.c" yymsp[-4].minor.yy122 = yylhsminor.yy122; break; case 488: /* when_then_expr ::= WHEN common_expression THEN common_expression */ #line 937 "sql.y" { yymsp[-3].minor.yy122 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122)); } -#line 7128 "sql.c" +#line 7133 "sql.c" break; case 490: /* case_when_else_opt ::= ELSE common_expression */ #line 940 "sql.y" { yymsp[-1].minor.yy122 = releaseRawExprNode(pCxt, yymsp[0].minor.yy122); } -#line 7133 "sql.c" +#line 7138 "sql.c" break; case 491: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ case 496: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==496); @@ -7140,7 +7145,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy416, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7143 "sql.c" +#line 7148 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 492: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ @@ -7150,7 +7155,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy122), releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7153 "sql.c" +#line 7158 "sql.c" yymsp[-4].minor.yy122 = yylhsminor.yy122; break; case 493: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ @@ -7160,7 +7165,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy122), releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7163 "sql.c" +#line 7168 "sql.c" yymsp[-5].minor.yy122 = yylhsminor.yy122; break; case 494: /* predicate ::= expr_or_subquery IS NULL */ @@ -7169,7 +7174,7 @@ static YYACTIONTYPE yy_reduce( SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), NULL)); } -#line 7172 "sql.c" +#line 7177 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 495: /* predicate ::= expr_or_subquery IS NOT NULL */ @@ -7178,78 +7183,78 @@ static YYACTIONTYPE yy_reduce( SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), NULL)); } -#line 7181 "sql.c" +#line 7186 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 497: /* compare_op ::= NK_LT */ #line 977 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_LOWER_THAN; } -#line 7187 "sql.c" +#line 7192 "sql.c" break; case 498: /* compare_op ::= NK_GT */ #line 978 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_GREATER_THAN; } -#line 7192 "sql.c" +#line 7197 "sql.c" break; case 499: /* compare_op ::= NK_LE */ #line 979 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_LOWER_EQUAL; } -#line 7197 "sql.c" +#line 7202 "sql.c" break; case 500: /* compare_op ::= NK_GE */ #line 980 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_GREATER_EQUAL; } -#line 7202 "sql.c" +#line 7207 "sql.c" break; case 501: /* compare_op ::= NK_NE */ #line 981 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_NOT_EQUAL; } -#line 7207 "sql.c" +#line 7212 "sql.c" break; case 502: /* compare_op ::= NK_EQ */ #line 982 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_EQUAL; } -#line 7212 "sql.c" +#line 7217 "sql.c" break; case 503: /* compare_op ::= LIKE */ #line 983 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_LIKE; } -#line 7217 "sql.c" +#line 7222 "sql.c" break; case 504: /* compare_op ::= NOT LIKE */ #line 984 "sql.y" { yymsp[-1].minor.yy416 = OP_TYPE_NOT_LIKE; } -#line 7222 "sql.c" +#line 7227 "sql.c" break; case 505: /* compare_op ::= MATCH */ #line 985 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_MATCH; } -#line 7227 "sql.c" +#line 7232 "sql.c" break; case 506: /* compare_op ::= NMATCH */ #line 986 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_NMATCH; } -#line 7232 "sql.c" +#line 7237 "sql.c" break; case 507: /* compare_op ::= CONTAINS */ #line 987 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_JSON_CONTAINS; } -#line 7237 "sql.c" +#line 7242 "sql.c" break; case 508: /* in_op ::= IN */ #line 991 "sql.y" { yymsp[0].minor.yy416 = OP_TYPE_IN; } -#line 7242 "sql.c" +#line 7247 "sql.c" break; case 509: /* in_op ::= NOT IN */ #line 992 "sql.y" { yymsp[-1].minor.yy416 = OP_TYPE_NOT_IN; } -#line 7247 "sql.c" +#line 7252 "sql.c" break; case 510: /* in_predicate_value ::= NK_LP literal_list NK_RP */ #line 994 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy298)); } -#line 7252 "sql.c" +#line 7257 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 512: /* boolean_value_expression ::= NOT boolean_primary */ @@ -7258,7 +7263,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy122), NULL)); } -#line 7261 "sql.c" +#line 7266 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 513: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ @@ -7268,7 +7273,7 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7271 "sql.c" +#line 7276 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 514: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ @@ -7278,64 +7283,64 @@ static YYACTIONTYPE yy_reduce( SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy122); yylhsminor.yy122 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7281 "sql.c" +#line 7286 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 522: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ #line 1027 "sql.y" { yylhsminor.yy122 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy122, yymsp[0].minor.yy122, NULL); } -#line 7287 "sql.c" +#line 7292 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 525: /* table_primary ::= table_name alias_opt */ #line 1033 "sql.y" { yylhsminor.yy122 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy203, &yymsp[0].minor.yy203); } -#line 7293 "sql.c" +#line 7298 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 526: /* table_primary ::= db_name NK_DOT table_name alias_opt */ #line 1034 "sql.y" { yylhsminor.yy122 = createRealTableNode(pCxt, &yymsp[-3].minor.yy203, &yymsp[-1].minor.yy203, &yymsp[0].minor.yy203); } -#line 7299 "sql.c" +#line 7304 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 527: /* table_primary ::= subquery alias_opt */ #line 1035 "sql.y" { yylhsminor.yy122 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122), &yymsp[0].minor.yy203); } -#line 7305 "sql.c" +#line 7310 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 529: /* alias_opt ::= */ #line 1040 "sql.y" { yymsp[1].minor.yy203 = nil_token; } -#line 7311 "sql.c" +#line 7316 "sql.c" break; case 531: /* alias_opt ::= AS table_alias */ #line 1042 "sql.y" { yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; } -#line 7316 "sql.c" +#line 7321 "sql.c" break; case 532: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ case 533: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==533); #line 1044 "sql.y" { yymsp[-2].minor.yy122 = yymsp[-1].minor.yy122; } -#line 7322 "sql.c" +#line 7327 "sql.c" break; case 534: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ #line 1049 "sql.y" { yylhsminor.yy122 = createJoinTableNode(pCxt, yymsp[-4].minor.yy498, yymsp[-5].minor.yy122, yymsp[-2].minor.yy122, yymsp[0].minor.yy122); } -#line 7327 "sql.c" +#line 7332 "sql.c" yymsp[-5].minor.yy122 = yylhsminor.yy122; break; case 535: /* join_type ::= */ #line 1053 "sql.y" { yymsp[1].minor.yy498 = JOIN_TYPE_INNER; } -#line 7333 "sql.c" +#line 7338 "sql.c" break; case 536: /* join_type ::= INNER */ #line 1054 "sql.y" { yymsp[0].minor.yy498 = JOIN_TYPE_INNER; } -#line 7338 "sql.c" +#line 7343 "sql.c" break; case 537: /* query_specification ::= SELECT hint_list set_quantifier_opt tag_mode_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ #line 1060 "sql.y" @@ -7351,42 +7356,42 @@ static YYACTIONTYPE yy_reduce( yymsp[-13].minor.yy122 = addEveryClause(pCxt, yymsp[-13].minor.yy122, yymsp[-4].minor.yy122); yymsp[-13].minor.yy122 = addFillClause(pCxt, yymsp[-13].minor.yy122, yymsp[-3].minor.yy122); } -#line 7354 "sql.c" +#line 7359 "sql.c" break; case 538: /* hint_list ::= */ #line 1075 "sql.y" { yymsp[1].minor.yy298 = createHintNodeList(pCxt, NULL); } -#line 7359 "sql.c" +#line 7364 "sql.c" break; case 539: /* hint_list ::= NK_HINT */ #line 1076 "sql.y" { yylhsminor.yy298 = createHintNodeList(pCxt, &yymsp[0].minor.yy0); } -#line 7364 "sql.c" +#line 7369 "sql.c" yymsp[0].minor.yy298 = yylhsminor.yy298; break; case 544: /* set_quantifier_opt ::= ALL */ #line 1087 "sql.y" { yymsp[0].minor.yy983 = false; } -#line 7370 "sql.c" +#line 7375 "sql.c" break; case 547: /* select_item ::= NK_STAR */ #line 1094 "sql.y" { yylhsminor.yy122 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } -#line 7375 "sql.c" +#line 7380 "sql.c" yymsp[0].minor.yy122 = yylhsminor.yy122; break; case 549: /* select_item ::= common_expression column_alias */ case 559: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==559); #line 1096 "sql.y" { yylhsminor.yy122 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122), &yymsp[0].minor.yy203); } -#line 7382 "sql.c" +#line 7387 "sql.c" yymsp[-1].minor.yy122 = yylhsminor.yy122; break; case 550: /* select_item ::= common_expression AS column_alias */ case 560: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==560); #line 1097 "sql.y" { yylhsminor.yy122 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), &yymsp[0].minor.yy203); } -#line 7389 "sql.c" +#line 7394 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 555: /* partition_by_clause_opt ::= PARTITION BY partition_list */ @@ -7394,99 +7399,99 @@ static YYACTIONTYPE yy_reduce( case 600: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==600); #line 1106 "sql.y" { yymsp[-2].minor.yy298 = yymsp[0].minor.yy298; } -#line 7397 "sql.c" +#line 7402 "sql.c" break; case 562: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ #line 1119 "sql.y" { yymsp[-5].minor.yy122 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7402 "sql.c" +#line 7407 "sql.c" break; case 563: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ #line 1120 "sql.y" { yymsp[-3].minor.yy122 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7407 "sql.c" +#line 7412 "sql.c" break; case 564: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ #line 1122 "sql.y" { yymsp[-5].minor.yy122 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), NULL, yymsp[-1].minor.yy122, yymsp[0].minor.yy122); } -#line 7412 "sql.c" +#line 7417 "sql.c" break; case 565: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ #line 1125 "sql.y" { yymsp[-7].minor.yy122 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy122), releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), yymsp[-1].minor.yy122, yymsp[0].minor.yy122); } -#line 7417 "sql.c" +#line 7422 "sql.c" break; case 566: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ #line 1127 "sql.y" { yymsp[-6].minor.yy122 = createEventWindowNode(pCxt, yymsp[-3].minor.yy122, yymsp[0].minor.yy122); } -#line 7422 "sql.c" +#line 7427 "sql.c" break; case 570: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ #line 1133 "sql.y" { yymsp[-3].minor.yy122 = createFillNode(pCxt, yymsp[-1].minor.yy312, NULL); } -#line 7427 "sql.c" +#line 7432 "sql.c" break; case 571: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ #line 1134 "sql.y" { yymsp[-5].minor.yy122 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy298)); } -#line 7432 "sql.c" +#line 7437 "sql.c" break; case 572: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ #line 1135 "sql.y" { yymsp[-5].minor.yy122 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy298)); } -#line 7437 "sql.c" +#line 7442 "sql.c" break; case 573: /* fill_mode ::= NONE */ #line 1139 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NONE; } -#line 7442 "sql.c" +#line 7447 "sql.c" break; case 574: /* fill_mode ::= PREV */ #line 1140 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_PREV; } -#line 7447 "sql.c" +#line 7452 "sql.c" break; case 575: /* fill_mode ::= NULL */ #line 1141 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NULL; } -#line 7452 "sql.c" +#line 7457 "sql.c" break; case 576: /* fill_mode ::= NULL_F */ #line 1142 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NULL_F; } -#line 7457 "sql.c" +#line 7462 "sql.c" break; case 577: /* fill_mode ::= LINEAR */ #line 1143 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_LINEAR; } -#line 7462 "sql.c" +#line 7467 "sql.c" break; case 578: /* fill_mode ::= NEXT */ #line 1144 "sql.y" { yymsp[0].minor.yy312 = FILL_MODE_NEXT; } -#line 7467 "sql.c" +#line 7472 "sql.c" break; case 581: /* group_by_list ::= expr_or_subquery */ #line 1153 "sql.y" { yylhsminor.yy298 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7472 "sql.c" +#line 7477 "sql.c" yymsp[0].minor.yy298 = yylhsminor.yy298; break; case 582: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ #line 1154 "sql.y" { yylhsminor.yy298 = addNodeToList(pCxt, yymsp[-2].minor.yy298, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy122))); } -#line 7478 "sql.c" +#line 7483 "sql.c" yymsp[-2].minor.yy298 = yylhsminor.yy298; break; case 586: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ #line 1161 "sql.y" { yymsp[-5].minor.yy122 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy122), releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7484 "sql.c" +#line 7489 "sql.c" break; case 587: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ #line 1163 "sql.y" { yymsp[-3].minor.yy122 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy122)); } -#line 7489 "sql.c" +#line 7494 "sql.c" break; case 590: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ #line 1170 "sql.y" @@ -7495,80 +7500,80 @@ static YYACTIONTYPE yy_reduce( yylhsminor.yy122 = addSlimitClause(pCxt, yylhsminor.yy122, yymsp[-1].minor.yy122); yylhsminor.yy122 = addLimitClause(pCxt, yylhsminor.yy122, yymsp[0].minor.yy122); } -#line 7498 "sql.c" +#line 7503 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 593: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ #line 1180 "sql.y" { yylhsminor.yy122 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy122, yymsp[0].minor.yy122); } -#line 7504 "sql.c" +#line 7509 "sql.c" yymsp[-3].minor.yy122 = yylhsminor.yy122; break; case 594: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ #line 1182 "sql.y" { yylhsminor.yy122 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy122, yymsp[0].minor.yy122); } -#line 7510 "sql.c" +#line 7515 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 602: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ case 606: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==606); #line 1196 "sql.y" { yymsp[-1].minor.yy122 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 7517 "sql.c" +#line 7522 "sql.c" break; case 603: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ case 607: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==607); #line 1197 "sql.y" { yymsp[-3].minor.yy122 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } -#line 7523 "sql.c" +#line 7528 "sql.c" break; case 604: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ case 608: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==608); #line 1198 "sql.y" { yymsp[-3].minor.yy122 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } -#line 7529 "sql.c" +#line 7534 "sql.c" break; case 609: /* subquery ::= NK_LP query_expression NK_RP */ #line 1206 "sql.y" { yylhsminor.yy122 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy122); } -#line 7534 "sql.c" +#line 7539 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 614: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ #line 1220 "sql.y" { yylhsminor.yy122 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy122), yymsp[-1].minor.yy626, yymsp[0].minor.yy877); } -#line 7540 "sql.c" +#line 7545 "sql.c" yymsp[-2].minor.yy122 = yylhsminor.yy122; break; case 615: /* ordering_specification_opt ::= */ #line 1224 "sql.y" { yymsp[1].minor.yy626 = ORDER_ASC; } -#line 7546 "sql.c" +#line 7551 "sql.c" break; case 616: /* ordering_specification_opt ::= ASC */ #line 1225 "sql.y" { yymsp[0].minor.yy626 = ORDER_ASC; } -#line 7551 "sql.c" +#line 7556 "sql.c" break; case 617: /* ordering_specification_opt ::= DESC */ #line 1226 "sql.y" { yymsp[0].minor.yy626 = ORDER_DESC; } -#line 7556 "sql.c" +#line 7561 "sql.c" break; case 618: /* null_ordering_opt ::= */ #line 1230 "sql.y" { yymsp[1].minor.yy877 = NULL_ORDER_DEFAULT; } -#line 7561 "sql.c" +#line 7566 "sql.c" break; case 619: /* null_ordering_opt ::= NULLS FIRST */ #line 1231 "sql.y" { yymsp[-1].minor.yy877 = NULL_ORDER_FIRST; } -#line 7566 "sql.c" +#line 7571 "sql.c" break; case 620: /* null_ordering_opt ::= NULLS LAST */ #line 1232 "sql.y" { yymsp[-1].minor.yy877 = NULL_ORDER_LAST; } -#line 7571 "sql.c" +#line 7576 "sql.c" break; default: break; @@ -7641,7 +7646,7 @@ static void yy_syntax_error( } else if (TSDB_CODE_PAR_DB_NOT_SPECIFIED == pCxt->errCode && TK_NK_FLOAT == TOKEN.type) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } -#line 7644 "sql.c" +#line 7649 "sql.c" /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE From dce29b367b9ad35e6cc350dd0f49a25ddaf610d4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 27 Sep 2023 13:46:14 +0800 Subject: [PATCH 02/47] fix: retore translater.c --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 81da234b06..959cd81e06 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1555,7 +1555,8 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN return TSDB_CODE_SUCCESS; } if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable || + QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } else { From 1aad905558e7118df00d5f884ef013fe7489ebdb Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 8 Oct 2023 15:55:01 +0800 Subject: [PATCH 03/47] fix: collection functions hash set distinct error --- source/libs/nodes/src/nodesUtilFuncs.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index bf5b6c8080..c5a1bfa599 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2033,7 +2033,6 @@ typedef struct SCollectFuncsCxt { char* tableAlias; FFuncClassifier classifier; SNodeList* pFuncs; - SHashObj* pFuncsSet; } SCollectFuncsCxt; static EDealRes collectFuncs(SNode* pNode, void* pContext) { @@ -2048,9 +2047,15 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { } } SExprNode* pExpr = (SExprNode*)pNode; - if (NULL == taosHashGet(pCxt->pFuncsSet, &pExpr, sizeof(SExprNode*))) { + bool bFound = false; + SNode* pn = NULL; + FOREACH(pn, pCxt->pFuncs) { + if (nodesEqualNode(pn, pNode)) { + bFound = true; + } + } + if (!bFound) { pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode)); - taosHashPut(pCxt->pFuncsSet, &pExpr, POINTER_BYTES, &pExpr, POINTER_BYTES); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } @@ -2077,12 +2082,10 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAl SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .classifier = classifier, .tableAlias = tableAlias, - .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs), - .pFuncsSet = taosHashInit(4, funcNodeHash, false, false)}; - if (NULL == cxt.pFuncs || NULL == cxt.pFuncsSet) { + .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs)}; + if (NULL == cxt.pFuncs) { return TSDB_CODE_OUT_OF_MEMORY; } - taosHashSetEqualFp(cxt.pFuncsSet, funcNodeEqual); *pFuncs = NULL; nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS == cxt.errCode) { @@ -2094,7 +2097,6 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAl } else { nodesDestroyList(cxt.pFuncs); } - taosHashCleanup(cxt.pFuncsSet); return cxt.errCode; } From 3f2d8905607a63ded0599bbe08ccde03d6e49d56 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 8 Oct 2023 16:06:15 +0800 Subject: [PATCH 04/47] feat:[TD-26056] add replay logic --- include/common/tmsg.h | 90 ++------------------- include/libs/executor/storageapi.h | 1 + include/util/taoserror.h | 2 + source/client/src/clientSml.c | 2 +- source/client/src/clientSmlJson.c | 40 +++++----- source/client/src/clientSmlLine.c | 2 +- source/client/src/clientTmq.c | 57 ++++++++------ source/common/src/tmsg.c | 33 ++------ source/dnode/mnode/impl/inc/mndDef.h | 26 ------- source/dnode/mnode/impl/src/mndConsumer.c | 23 +++++- source/dnode/mnode/impl/src/mndDump.c | 20 +---- source/dnode/mnode/impl/src/mndTopic.c | 22 ------ source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/inc/tq.h | 11 +-- source/dnode/vnode/src/tq/tq.c | 3 + source/dnode/vnode/src/tq/tqOffset.c | 13 ++-- source/dnode/vnode/src/tq/tqRead.c | 8 +- source/dnode/vnode/src/tq/tqScan.c | 95 +++++++++++++++++------ source/dnode/vnode/src/tq/tqUtil.c | 4 +- source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + source/libs/executor/src/scanoperator.c | 3 + source/util/src/terror.c | 2 + utils/test/c/tmqOffset.c | 9 +-- 23 files changed, 192 insertions(+), 278 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index af44184de6..90147a3c31 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2295,17 +2295,6 @@ int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateS int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq); void tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq); -typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - int64_t streamId; - char* sql; - char* executorMsg; -} SMVCreateStreamReq, SMSCreateStreamReq; - -typedef struct { - int64_t streamId; -} SMVCreateStreamRsp, SMSCreateStreamRsp; - enum { TOPIC_SUB_TYPE__DB = 1, TOPIC_SUB_TYPE__TABLE, @@ -2327,16 +2316,9 @@ int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTo int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq); void tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq); -typedef struct { - int64_t topicId; -} SCMCreateTopicRsp; - -int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp); -int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp); - typedef struct { int64_t consumerId; -} SMqConsumerLostMsg, SMqConsumerRecoverMsg, SMqConsumerClearMsg; +} SMqConsumerRecoverMsg, SMqConsumerClearMsg; typedef struct { int64_t consumerId; @@ -2348,6 +2330,7 @@ typedef struct { int8_t autoCommit; int32_t autoCommitInterval; int8_t resetOffsetCfg; + int8_t enableReplay; } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -2367,6 +2350,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc tlen += taosEncodeFixedI8(buf, pReq->autoCommit); tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval); tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg); + tlen += taosEncodeFixedI8(buf, pReq->enableReplay); return tlen; } @@ -2390,71 +2374,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI8(buf, &pReq->autoCommit); buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval); buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); - return buf; -} - -typedef struct SMqSubTopic { - int32_t vgId; - int64_t topicId; - SEpSet epSet; -} SMqSubTopic; - -typedef struct { - int32_t topicNum; - SMqSubTopic topics[]; -} SCMSubscribeRsp; - -static FORCE_INLINE int32_t tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pRsp->topicNum); - for (int32_t i = 0; i < pRsp->topicNum; i++) { - tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId); - tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId); - tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet); - } - return tlen; -} - -static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { - buf = taosDecodeFixedI32(buf, &pRsp->topicNum); - for (int32_t i = 0; i < pRsp->topicNum; i++) { - buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId); - buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId); - buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet); - } - return buf; -} - -typedef struct { - int64_t topicId; - int64_t consumerId; - int64_t consumerGroupId; - int64_t offset; - char* sql; - char* logicalPlan; - char* physicalPlan; -} SMVSubscribeReq; - -static FORCE_INLINE int32_t tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->topicId); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId); - tlen += taosEncodeFixedI64(buf, pReq->offset); - tlen += taosEncodeString(buf, pReq->sql); - tlen += taosEncodeString(buf, pReq->logicalPlan); - tlen += taosEncodeString(buf, pReq->physicalPlan); - return tlen; -} - -static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->topicId); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeFixedI64(buf, &pReq->consumerGroupId); - buf = taosDecodeFixedI64(buf, &pReq->offset); - buf = taosDecodeString(buf, &pReq->sql); - buf = taosDecodeString(buf, &pReq->logicalPlan); - buf = taosDecodeString(buf, &pReq->physicalPlan); + buf = taosDecodeFixedI8(buf, &pReq->enableReplay); return buf; } @@ -3534,6 +3454,7 @@ typedef struct { int64_t consumerId; int64_t timeout; STqOffsetVal reqOffset; + int8_t enableReplay; } SMqPollReq; int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); @@ -3593,6 +3514,7 @@ typedef struct { SArray* blockData; SArray* blockTbName; SArray* blockSchema; + int64_t sleepTime; } SMqDataRsp; int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0a240dd8f5..fa110d66e2 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -222,6 +222,7 @@ typedef struct SStoreTqReader { bool (*tqReaderNextBlockInWal)(); bool (*tqNextBlockImpl)(); // todo remove it SSDataBlock* (*tqGetResultBlock)(); + int64_t (*tqGetResultBlockTime)(); void (*tqReaderSetColIdList)(); int32_t (*tqReaderSetQueryTableList)(); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 39bf2b5681..eb991379c4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -792,6 +792,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_NEED_INITIALIZED TAOS_DEF_ERROR_CODE(0, 0x4010) #define TSDB_CODE_TMQ_NO_COMMITTED TAOS_DEF_ERROR_CODE(0, 0x4011) #define TSDB_CODE_TMQ_SAME_COMMITTED_VALUE TAOS_DEF_ERROR_CODE(0, 0x4012) +#define TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP TAOS_DEF_ERROR_CODE(0, 0x4013) +#define TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x4014) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 10f8b89f4d..91c21fe344 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -683,7 +683,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { taosHashCleanup(hashTmp); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } } taosHashCleanup(hashTmp); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index f9076112c4..5e656c71a7 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -256,7 +256,8 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { } if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) return -1; + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); + return TSDB_CODE_TSC_INVALID_JSON; } char *sTmp = *start; @@ -367,7 +368,8 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || element->measure == NULL || element->timestamp == NULL) { - uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM) return -1; + uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM); + return TSDB_CODE_TSC_INVALID_JSON; } return 0; } @@ -381,7 +383,8 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) return -1; + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); + return TSDB_CODE_TSC_INVALID_JSON; } if ((*start)[1] == 'm') { @@ -448,7 +451,8 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { } if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { - uError("elements != %d", OTD_JSON_FIELDS_NUM) return -1; + uError("elements != %d", OTD_JSON_FIELDS_NUM); + return TSDB_CODE_TSC_INVALID_JSON; } return 0; } @@ -477,7 +481,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { } if (*marks[i] == NULL) { uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } } return TSDB_CODE_SUCCESS; @@ -816,25 +820,25 @@ static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPr int32_t size = cJSON_GetArraySize(root); if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cJSON *value = cJSON_GetObjectItem(root, "value"); if (unlikely(!cJSON_IsNumber(value))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } cJSON *type = cJSON_GetObjectItem(root, "type"); if (unlikely(!cJSON_IsString(type))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } double timeDouble = value->valuedouble; if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } if (timeDouble == 0) { @@ -849,32 +853,28 @@ static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPr size_t typeLen = strlen(type->valuestring); if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { // seconds - int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) { return tsInt64 * smlFactorS[toPrecision]; } - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { switch (type->valuestring[0]) { case 'm': case 'M': // milliseconds return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); - break; case 'u': case 'U': // microseconds return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); - break; case 'n': case 'N': return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); - break; default: - return -1; + return TSDB_CODE_TSC_INVALID_JSON_TYPE; } } else { - return -1; + return TSDB_CODE_TSC_INVALID_JSON_TYPE; } } @@ -895,7 +895,7 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { double timeDouble = timestamp->valuedouble; if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } if (unlikely(timeDouble < 0)) { @@ -911,14 +911,14 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { if (unlikely(fromPrecision == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } int64_t tsInt64 = timeDouble; if (fromPrecision == TSDB_TIME_PRECISION_SECONDS) { if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) { return tsInt64 * smlFactorS[toPrecision]; } - return -1; + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } else { return convertTimePrecision(timeDouble, fromPrecision, toPrecision); } @@ -926,7 +926,7 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { return smlParseTSFromJSONObj(info, timestamp, toPrecision); } else { smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; + return TSDB_CODE_TSC_INVALID_JSON; } } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index a565fb1a21..006475654a 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -70,7 +70,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); if (unlikely(ts == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } return ts; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 6ee5508048..50b8eb1eca 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -63,7 +63,7 @@ struct tmq_conf_t { int8_t resetOffset; int8_t withTbName; int8_t snapEnable; - int32_t snapBatchSize; + int8_t replayEnable; bool hbBgEnable; uint16_t port; int32_t autoCommitInterval; @@ -83,6 +83,7 @@ struct tmq_t { int8_t autoCommit; int32_t autoCommitInterval; int8_t resetOffsetCfg; + int8_t replayEnable; uint64_t consumerId; bool hbBgEnable; tmq_commit_cb* commitCb; @@ -92,19 +93,13 @@ struct tmq_t { SRWLatch lock; int8_t status; int32_t epoch; -#if 0 - int8_t epStatus; - int32_t epSkipCnt; -#endif // poll info int64_t pollCnt; int64_t totalRows; -// bool needReportOffsetRows; // timer tmr_h hbLiveTimer; tmr_h epTimer; - tmr_h reportTimer; tmr_h commitTimer; STscObj* pTscObj; // connection SArray* clientTopics; // SArray @@ -152,6 +147,8 @@ typedef struct { int32_t vgStatus; int32_t vgSkipCnt; // here used to mark the slow vgroups int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int64_t blockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int64_t blockSleepForReplay; // once empty block is received, idle for ignoreCnt then start to poll data bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. SEpSet epSet; } SMqClientVg; @@ -360,24 +357,6 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } - if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) { - conf->snapBatchSize = taosStr2int64(value); - return TMQ_CONF_OK; - } - -// if (strcasecmp(key, "enable.heartbeat.background") == 0) { - // if (strcasecmp(value, "true") == 0) { - // conf->hbBgEnable = true; - // return TMQ_CONF_OK; - // } else if (strcasecmp(value, "false") == 0) { - // conf->hbBgEnable = false; - // return TMQ_CONF_OK; - // } else { -// tscError("the default value of enable.heartbeat.background is true, can not be seted"); -// return TMQ_CONF_INVALID; - // } -// } - if (strcasecmp(key, "td.connect.ip") == 0) { conf->ip = taosStrdup(value); return TMQ_CONF_OK; @@ -398,6 +377,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } + if (strcasecmp(key, "enable.replay") == 0) { + if (strcasecmp(value, "true") == 0) { + conf->replayEnable = true; + return TMQ_CONF_OK; + } else if (strcasecmp(value, "false") == 0) { + conf->replayEnable = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } + if (strcasecmp(key, "td.connect.db") == 0) { return TMQ_CONF_OK; } @@ -1075,6 +1066,10 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + pTmq->replayEnable = conf->replayEnable; + if(conf->replayEnable){ + pTmq->autoCommit = false; + } taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1424,6 +1419,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, .vgSkipCnt = 0, .emptyBlockReceiveTs = 0, + .blockReceiveTs = 0, + .blockSleepForReplay = 0, .numOfRows = pInfo ? pInfo->numOfRows : 0, }; @@ -1695,6 +1692,12 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { continue; } + if (tmq->replayEnable && taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) { // less than 10ms + tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId, + tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); + continue; + } + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); @@ -1816,6 +1819,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; + if(tmq->replayEnable){ + pVg->blockReceiveTs = taosGetTimestampMs(); + pVg->blockSleepForReplay = pRsp->rsp.sleepTime; + } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index fd39ae98d9..7d285434d9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4352,31 +4352,6 @@ void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { } } -int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->topicId) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->topicId) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -5983,6 +5958,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1; if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1; + if (tEncodeI8(&encoder, pReq->enableReplay) < 0) return -1; tEndEncode(&encoder); @@ -6019,6 +5995,10 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1; if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->enableReplay) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -7795,6 +7775,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { } } } + if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1; return 0; } @@ -7840,6 +7821,8 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } } } + if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + return 0; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 844e69e659..1715f56091 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -490,32 +490,6 @@ typedef struct { char filterTb[TSDB_TABLE_NAME_LEN]; } SShowObj; -typedef struct { - int64_t id; - int8_t type; - int8_t replica; - int16_t numOfColumns; - int32_t rowSize; - int32_t numOfRows; - int32_t numOfReads; - int32_t payloadLen; - void* pIter; - SMnode* pMnode; - char db[TSDB_DB_FNAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; - char payload[]; -} SSysTableRetrieveObj; - -typedef struct { - char key[TSDB_PARTITION_KEY_LEN]; - int64_t dbUid; - int64_t offset; -} SMqOffsetObj; - -int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset); -void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset); - typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7273e13317..05156e1427 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -18,6 +18,7 @@ #include "mndPrivilege.h" #include "mndVgroup.h" #include "mndShow.h" +#include "mndDb.h" #include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" @@ -124,7 +125,7 @@ void mndRebCntDec() { } } -static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser) { +static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { @@ -139,6 +140,22 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * return -1; } + if(enableReplay){ + if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){ + return TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; + }else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { + SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); + if (pDb == NULL) { + mndReleaseTopic(pMnode, pTopic); + return -1; + } + if (pDb->cfg.numOfVgroups != 1) { + return TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; + } + mndReleaseDb(pMnode, pDb); + } + } + mndTransSetDbName(pTrans, pOneTopic, NULL); if(mndTransCheckConflict(pMnode, pTrans) != 0){ mndReleaseTopic(pMnode, pTopic); @@ -177,7 +194,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { if (pTrans == NULL) { goto FAIL; } - if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user) != 0){ + if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0){ goto FAIL; } @@ -697,7 +714,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto _over; } - code = validateTopics(pTrans, pTopicList, pMnode, pMsg->info.conn.user); + code = validateTopics(pTrans, pTopicList, pMnode, pMsg->info.conn.user, subscribe.enableReplay); if (code != TSDB_CODE_SUCCESS) { goto _over; } diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 62b5cb00e6..481495cbe5 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -330,24 +330,6 @@ void dumpSubscribe(SSdb *pSdb, SJson *json) { } } -void dumpOffset(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonAddArrayToObject(json, "offsets"); - - while (1) { - SMqOffsetObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToArray(items, item); - tjsonAddStringToObject(item, "key", pObj->key); - tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); - tjsonAddStringToObject(item, "offset", i642str(pObj->offset)); - sdbRelease(pSdb, pObj); - } -} - void dumpStream(SSdb *pSdb, SJson *json) { void *pIter = NULL; SJson *items = tjsonAddArrayToObject(json, "streams"); @@ -608,7 +590,7 @@ void mndDumpSdb() { dumpTopic(pSdb, json); dumpConsumer(pSdb, json); dumpSubscribe(pSdb, json); - dumpOffset(pSdb, json); +// dumpOffset(pSdb, json); dumpStream(pSdb, json); dumpAcct(pSdb, json); dumpAuth(pSdb, json); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 94fd6027c0..4b5cfc830f 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -298,11 +298,6 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->version, pNewTopic->version); - /*taosWLockLatch(&pOldTopic->lock);*/ - - // TODO handle update - - /*taosWUnLockLatch(&pOldTopic->lock);*/ return 0; } @@ -320,23 +315,6 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { sdbRelease(pSdb, pTopic); } -static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { - int32_t contLen = sizeof(SDDropTopicReq); - - SDDropTopicReq *pDrop = taosMemoryCalloc(1, contLen); - if (pDrop == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pDrop->head.contLen = htonl(contLen); - pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pTopic->name, TSDB_TOPIC_FNAME_LEN); - pDrop->tuid = htobe64(pTopic->uid); - - return pDrop; -} - static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { terrno = TSDB_CODE_MND_INVALID_TOPIC; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e15f5f911d..ee0f0e2eeb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -213,7 +213,7 @@ typedef struct STqReader { SPackedData msg; SSubmitReq2 submit; int32_t nextBlk; - int64_t lastBlkUid; + int64_t lastTs; SWalReader *pWalReader; SMeta *pVnodeMeta; SHashObj *tbIdHash; @@ -241,6 +241,7 @@ bool tqNextBlockInWal(STqReader *pReader, const char *idstr); bool tqNextBlockImpl(STqReader *pReader, const char *idstr); SWalReader *tqGetWalReader(STqReader *pReader); SSDataBlock *tqGetResultBlock(STqReader *pReader); +int64_t tqGetResultBlockTime(STqReader *pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ee96e602d8..5d4ea8699b 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -47,6 +47,7 @@ typedef struct STqOffsetStore STqOffsetStore; // tqPush #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) #define STREAM_EXEC_TASK_STATUS_CHECK_ID (-2) +#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) // tqExec typedef struct { @@ -91,6 +92,10 @@ typedef struct { STqExecHandle execHandle; // exec SRpcMsg* msg; tq_handle_status status; + + // for replay + SSDataBlock* block; + int64_t blockTime; } STqHandle; struct STQ { @@ -108,17 +113,13 @@ struct STQ { SStreamMeta* pStreamMeta; }; -typedef struct { - int32_t size; -} STqOffsetHead; - int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); void tqDestroyTqHandle(void* data); // tqRead int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); -int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); +int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index adf3abe4d9..2fda70bcc1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -82,6 +82,9 @@ void tqDestroyTqHandle(void* data) { taosMemoryFree(pData->msg); pData->msg = NULL; } + if (pData->block != NULL){ + blockDataDestroy(pData->block); + } } static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 11bb737225..5e67f3c3ac 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -37,11 +37,9 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { int32_t vgId = TD_VID(pStore->pTq->pVnode); int64_t code = 0; - - STqOffsetHead head = {0}; - + int32_t size = 0; while (1) { - if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if (code == 0) { break; } else { @@ -49,7 +47,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { } } - int32_t size = htonl(head.size); void* pMemBuf = taosMemoryCalloc(1, size); if (pMemBuf == NULL) { tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); @@ -175,11 +172,11 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { return -1; } - int32_t totLen = sizeof(STqOffsetHead) + bodyLen; + int32_t totLen = INT_BYTES + bodyLen; void* buf = taosMemoryCalloc(1, totLen); - void* abuf = POINTER_SHIFT(buf, sizeof(STqOffsetHead)); + void* abuf = POINTER_SHIFT(buf, INT_BYTES); - ((STqOffsetHead*)buf)->size = htonl(bodyLen); + *(int32_t*)buf = bodyLen; SEncoder encoder; tEncoderInit(&encoder, abuf, bodyLen); tEncodeSTqOffset(&encoder, pOffset); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 4d470ee5b6..65914cc70e 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -475,6 +475,10 @@ SSDataBlock* tqGetResultBlock (STqReader* pReader) { return pReader->pResBlock; } +int64_t tqGetResultBlockTime(STqReader *pReader){ + return pReader->lastTs; +} + bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { return false; @@ -641,7 +645,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; - pReader->lastBlkUid = uid; + pReader->lastTs = pSubmitTbData->ctimeMs; pBlock->info.id.uid = uid; pBlock->info.version = pReader->msg.ver; @@ -783,9 +787,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas } int32_t sversion = pSubmitTbData->sver; - int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; - pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index cbe3ffee9e..705fb86fab 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -64,7 +64,23 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in return 0; } -int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { +int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res){ + uint64_t ts = 0; + qStreamSetOpen(task); + + tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); + int32_t code = qExecTask(task, res, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; + return -1; + } + + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, *res); + return 0; +} + +int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) { const int32_t MAX_ROWS_TO_RETURN = 4096; int32_t vgId = TD_VID(pTq->pVnode); @@ -80,34 +96,63 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs while (1) { SSDataBlock* pDataBlock = NULL; - uint64_t ts = 0; - qStreamSetOpen(task); - - tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - code = qExecTask(task, &pDataBlock, &ts); - if (code != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); - terrno = code; - return -1; - } - - tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, - pDataBlock); - // current scan should be stopped asap, since the rebalance occurs. - if (pDataBlock == NULL) { - break; - } - - code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d, failed to add block to rsp msg", vgId); + code = getDataBlock(task, pHandle, vgId, &pDataBlock); + if (code != 0){ return code; } - pRsp->blockNum++; - totalRows += pDataBlock->info.rows; - if (totalRows >= MAX_ROWS_TO_RETURN) { + if(pRequest->enableReplay){ + if(IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL){ + blockDataDestroy(pHandle->block); + pHandle->block = NULL; + } + if(pHandle->block == NULL){ + if (pDataBlock == NULL) { + break; + } + STqOffsetVal offset = {0}; + qStreamExtractOffset(task, &offset); + pHandle->block = createDataBlock(); + copyDataBlock(pHandle->block, pDataBlock); + pHandle->blockTime = offset.ts; + code = getDataBlock(task, pHandle, vgId, &pDataBlock); + if (code != 0){ + return code; + } + } + + code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, failed to add block to rsp msg", vgId); + return code; + } + + pRsp->blockNum++; + if (pDataBlock == NULL) { + break; + } + copyDataBlock(pHandle->block, pDataBlock); + + STqOffsetVal offset = {0}; + qStreamExtractOffset(task, &offset); + pRsp->sleepTime = offset.ts - pHandle->blockTime; + pHandle->blockTime = offset.ts; break; + }else{ + if (pDataBlock == NULL) { + break; + } + code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, failed to add block to rsp msg", vgId); + return code; + } + + pRsp->blockNum++; + totalRows += pDataBlock->info.rows; + if (totalRows >= MAX_ROWS_TO_RETURN) { + break; + } } } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 04695c1f63..a4c3d395e3 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -15,8 +15,6 @@ #include "tq.h" -#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) - static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); @@ -142,7 +140,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, tqInitDataRsp(&dataRsp, *pOffset); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index c72ecd4824..a6a2e128d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -131,6 +131,7 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqGetResultBlock = tqGetResultBlock; pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; + pTq->tqGetResultBlockTime = tqGetResultBlockTime; } void initStateStoreAPI(SStateStore* pStore) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 474128007a..913e246f63 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1862,6 +1862,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { // curVersion move to next tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion); + // use ts to pass time when replay, because ts not used if type is log + pTaskInfo->streamInfo.currentOffset.ts = pAPI->tqReaderFn.tqGetResultBlockTime(pInfo->tqReader); + if (hasResult) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9832720994..dd51b99cd1 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -654,6 +654,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE, "Topic num out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE, "Group num out of range 100") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SAME_COMMITTED_VALUE, "Same committed value") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP, "Replay need only one vgroup if subscribe super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled if subscribe db or stable") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") diff --git a/utils/test/c/tmqOffset.c b/utils/test/c/tmqOffset.c index 7225cb87bd..9699e71f24 100644 --- a/utils/test/c/tmqOffset.c +++ b/utils/test/c/tmqOffset.c @@ -7,18 +7,14 @@ #include "tlog.h" #include "tmsg.h" -typedef struct { - int32_t size; -} STqOffsetHead; - int32_t tqOffsetRestoreFromFile(const char* fname) { TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile != NULL) { - STqOffsetHead head = {0}; int32_t code; while (1) { - if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + int32_t size = 0; + if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if (code == 0) { break; } else { @@ -26,7 +22,6 @@ int32_t tqOffsetRestoreFromFile(const char* fname) { return -1; } } - int32_t size = htonl(head.size); void* memBuf = taosMemoryCalloc(1, size); if (memBuf == NULL) { printf("memBuf == NULL\n"); From 5f7b6f19baeac0a5d4d30895f80e11ece6246328 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 8 Oct 2023 19:05:59 +0800 Subject: [PATCH 05/47] feat:[TD-26056] add replay logic --- source/client/src/clientTmq.c | 1 + source/dnode/vnode/inc/vnode.h | 3 +- source/libs/executor/src/projectoperator.c | 3 +- tests/system-test/7-tmq/replay.py | 320 +++++++++++++++++++++ 4 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/7-tmq/replay.py diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 50b8eb1eca..252959ecd8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1532,6 +1532,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->head.vgId = pVg->vgId; pReq->useSnapshot = tmq->useSnapshot; pReq->reqId = generateRequestId(); + pReq->enableReplay = tmq->replayEnable; } SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 122f6733fb..787b717015 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -216,7 +216,7 @@ typedef struct STqReader { SPackedData msg; SSubmitReq2 submit; int32_t nextBlk; - int64_t lastTs; + int64_t lastBlkUid; SWalReader *pWalReader; SMeta *pVnodeMeta; SHashObj *tbIdHash; @@ -226,6 +226,7 @@ typedef struct STqReader { int64_t cachedSchemaUid; SSchemaWrapper *pSchemaWrapper; SSDataBlock *pResBlock; + int64_t lastTs; } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 00b246afad..227960ab22 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -110,7 +110,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder; - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM || pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { pInfo->mergeDataBlocks = false; } else { if (!pProjPhyNode->ignoreGroupId) { @@ -330,6 +330,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } + qDebug("project return %d", pProjectInfo->mergeDataBlocks); if (pProjectInfo->mergeDataBlocks) { if (pRes->info.rows > 0) { pFinalRes->info.id.groupId = 0; // clear groupId diff --git a/tests/system-test/7-tmq/replay.py b/tests/system-test/7-tmq/replay.py new file mode 100644 index 0000000000..7eee6743a7 --- /dev/null +++ b/tests/system-test/7-tmq/replay.py @@ -0,0 +1,320 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + time.sleep(1) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"], \ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase8(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 8: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db8', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 1, \ + 'rowsPerTbl': 10, \ + 'batchNum': 1, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest,\ + enable.replay:true' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume 0 processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume 0 result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + # tdLog.info("start consume 1 processor") + # self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + # tdLog.sleep(2) + # + # tdLog.info("start one new thread to insert data") + # parameterDict['actionType'] = actionType.INSERT_DATA + # prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + # prepareEnvThread.start() + # prepareEnvThread.join() + # + # tdLog.info("start to check consume 0 and 1 result") + # expectRows = 2 + # resultList = self.selectConsumeResult(expectRows) + # totalConsumeRows = 0 + # for i in range(expectRows): + # totalConsumeRows += resultList[i] + # + # if totalConsumeRows != expectrowcnt: + # tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + # tdLog.exit("tmq consume rows error!") + # + # tdLog.info("start consume 2 processor") + # self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + # tdLog.sleep(2) + # + # tdLog.info("start one new thread to insert data") + # parameterDict['actionType'] = actionType.INSERT_DATA + # prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + # prepareEnvThread.start() + # prepareEnvThread.join() + # + # tdLog.info("start to check consume 0 and 1 and 2 result") + # expectRows = 3 + # resultList = self.selectConsumeResult(expectRows) + # totalConsumeRows = 0 + # for i in range(expectRows): + # totalConsumeRows += resultList[i] + # + # if totalConsumeRows != expectrowcnt*2: + # tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) + # tdLog.exit("tmq consume rows error!") + # + # tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 8 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase8(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 6ad518d61de8f142e4160d4aff15b15d43bd0ce2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 9 Oct 2023 13:21:32 +0800 Subject: [PATCH 06/47] enhance: pseduo column coverted to column when necessary --- source/libs/parser/src/parTranslater.c | 96 ++++++++++++++++---------- 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0fc57d3729..2a55a5fcd3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1571,25 +1571,6 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { return TSDB_CODE_SUCCESS; } -static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } - } else { - SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); - STableNode* pTable = NULL; - pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsIndefiniteRowsFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1726,20 +1707,6 @@ static int32_t translateForbidFillFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } -static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); - } - if (beforeWindow(pCxt->currClause)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", - pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsForbidStreamFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1978,10 +1945,65 @@ static int32_t rewriteSystemInfoFunc(STranslateContext* pCxt, SNode** pNode) { return TSDB_CODE_PAR_INTERNAL_ERROR; } -static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { +static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode** ppNode) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (pCol == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SExprNode* pOldExpr = (SExprNode*)(*ppNode); + pCol->node.resType = pOldExpr->resType; + strcpy(pCol->node.aliasName, pOldExpr->aliasName); + strcpy(pCol->node.userAlias, pOldExpr->userAlias); + strcpy(pCol->colName, pOldExpr->aliasName); + + nodesDestroyNode(*ppNode); + *ppNode = (SNode*)pCol; + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (!isSelectStmt(pCxt->pCurrStmt)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); + } + if (((SSelectStmt*)pCxt->pCurrStmt)->pWindow && beforeWindow(pCxt->currClause)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", + pFunc->functionName); + } + if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { + replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + return translateColumn(pCxt, (SColumnNode**)ppNode); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateScanPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (0 == LIST_LENGTH(pFunc->pParameterList)) { + if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } + if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + return translateColumn(pCxt, (SColumnNode**)ppNode); + } + } else { + SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + STableNode* pTable = NULL; + pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); + if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { - code = translateScanPseudoColumnFunc(pCxt, pFunc); + code = translateScanPseudoColumnFunc2(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateIndefiniteRowsFunc(pCxt, pFunc); @@ -1990,7 +2012,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p code = translateForbidFillFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindowPseudoColumnFunc(pCxt, pFunc); + code = translateWindowPseudoColumnFunc2(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); @@ -2082,7 +2104,7 @@ static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pF if (fmIsClientPseudoColumnFunc((*pFunc)->funcId)) { return rewriteClientPseudoColumnFunc(pCxt, (SNode**)pFunc); } - return translateNormalFunction(pCxt, *pFunc); + return translateNormalFunction(pCxt, (SNode**)pFunc); } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) { From 56534a7b0d47b2601a960dee5a985f97b5222dc3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 9 Oct 2023 16:22:44 +0800 Subject: [PATCH 07/47] enhance: scan and window pseudo column --- source/libs/parser/src/parTranslater.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b341e4e5eb..0228a7fded 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1934,6 +1934,9 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (!isSelectStmt(pCxt->pCurrStmt)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); } @@ -1942,27 +1945,38 @@ static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** pFunc->functionName); } if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - return translateColumn(pCxt, (SColumnNode**)ppNode); + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + return pCxt->errCode; } return TSDB_CODE_SUCCESS; } static int32_t translateScanPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } if (0 == LIST_LENGTH(pFunc->pParameterList)) { if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { - replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - return translateColumn(pCxt, (SColumnNode**)ppNode); + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + return pCxt->errCode; } } else { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); STableNode* pTable = NULL; pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } From 44c3bc99e09c939eac667ed295b90be3724eefe5 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 9 Oct 2023 16:56:21 +0800 Subject: [PATCH 08/47] feat: add interp pseudo column conversion to column --- source/libs/parser/src/parTranslater.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0228a7fded..4e0979ddaa 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -263,6 +263,8 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMet static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); +static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode** ppNode); + static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; } static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } @@ -1619,7 +1621,8 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } -static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } @@ -1631,6 +1634,24 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctio return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, "%s is not allowed in where clause", pFunc->functionName); } + + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + SNode* pNode = NULL; + bool bFound = false; + FOREACH(pNode, pSelect->pProjectionList) { + if (nodeType(pNode) == QUERY_NODE_FUNCTION && strcasecmp(((SFunctionNode*)pNode)->functionName, "interp") == 0) { + bFound = true; + break; + } + } + if (!bFound) { + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + return pCxt->errCode; + } return TSDB_CODE_SUCCESS; } @@ -2017,7 +2038,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateInterpFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateInterpPseudoColumnFunc(pCxt, pFunc); + code = translateInterpPseudoColumnFunc(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); From 33045e63ae88bd4335ec203f6e6e22a619684498 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 17:35:40 +0800 Subject: [PATCH 09/47] feat:[TD-26056] add replay logic --- source/client/src/clientTmq.c | 17 ++ source/dnode/vnode/src/tq/tqScan.c | 21 +- source/dnode/vnode/src/tq/tqUtil.c | 6 +- source/libs/executor/src/projectoperator.c | 1 - tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/replay.py | 33 +-- tests/system-test/7-tmq/tmq_replay.py | 39 +++ utils/test/c/CMakeLists.txt | 9 + utils/test/c/replay_test.c | 323 +++++++++++++++++++++ utils/test/c/tmqSim.c | 3 +- 10 files changed, 418 insertions(+), 35 deletions(-) create mode 100644 tests/system-test/7-tmq/tmq_replay.py create mode 100644 utils/test/c/replay_test.c diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 252959ecd8..e398441979 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -724,6 +724,17 @@ void tmqAssignAskEpTask(void* param, void* tmrId) { taosMemoryFree(param); } +void tmqReplayTask(void* param, void* tmrId) { + int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if(tmq == NULL) goto END; + + tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); +END: + taosMemoryFree(param); +} + void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT); @@ -1144,6 +1155,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.autoCommit = tmq->autoCommit; req.autoCommitInterval = tmq->autoCommitInterval; req.resetOffsetCfg = tmq->resetOffsetCfg; + req.enableReplay = tmq->replayEnable; for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1823,6 +1835,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { if(tmq->replayEnable){ pVg->blockReceiveTs = taosGetTimestampMs(); pVg->blockSleepForReplay = pRsp->rsp.sleepTime; + if(pVg->blockSleepForReplay > 0){ + int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t)); + *pRefId1 = tmq->refId; + taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, pRefId1, tmqMgmt.timer); + } } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 705fb86fab..01866ef893 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -112,8 +112,9 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* } STqOffsetVal offset = {0}; qStreamExtractOffset(task, &offset); - pHandle->block = createDataBlock(); - copyDataBlock(pHandle->block, pDataBlock); + pHandle->block = createOneDataBlock(pDataBlock, true); +// pHandle->block = createDataBlock(); +// copyDataBlock(pHandle->block, pDataBlock); pHandle->blockTime = offset.ts; code = getDataBlock(task, pHandle, vgId, &pDataBlock); if (code != 0){ @@ -129,14 +130,16 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pRsp->blockNum++; if (pDataBlock == NULL) { - break; - } - copyDataBlock(pHandle->block, pDataBlock); + blockDataDestroy(pHandle->block); + pHandle->block = NULL; + }else{ + copyDataBlock(pHandle->block, pDataBlock); - STqOffsetVal offset = {0}; - qStreamExtractOffset(task, &offset); - pRsp->sleepTime = offset.ts - pHandle->blockTime; - pHandle->blockTime = offset.ts; + STqOffsetVal offset = {0}; + qStreamExtractOffset(task, &offset); + pRsp->sleepTime = offset.ts - pHandle->blockTime; + pHandle->blockTime = offset.ts; + } break; }else{ if (pDataBlock == NULL) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index a4c3d395e3..215f8d3cb2 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -40,11 +40,11 @@ void tqUpdateNodeStage(STQ* pTq) { tqDebug("vgId:%d update the meta stage to be:%"PRId64, pTq->pStreamMeta->vgId, pTq->pStreamMeta->stage); } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset, bool withTbName) { pRsp->reqOffset = pOffset; pRsp->rspOffset = pOffset; - pRsp->withTbName = 1; + pRsp->withTbName = withTbName; pRsp->withSchema = 1; pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -177,7 +177,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t vgId = TD_VID(pTq->pVnode); SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, *offset); + tqInitTaosxRsp(&taosxRsp, *offset, pRequest->withTbName); if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 227960ab22..f60890ecca 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -330,7 +330,6 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } - qDebug("project return %d", pProjectInfo->mergeDataBlocks); if (pProjectInfo->mergeDataBlocks) { if (pRes->info.rows > 0) { pFinalRes->info.id.groupId = 0; // clear groupId diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 36b8fded81..b5856fea63 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -159,6 +159,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDataPrecisionUnit.py diff --git a/tests/system-test/7-tmq/replay.py b/tests/system-test/7-tmq/replay.py index 7eee6743a7..bbda8600fb 100644 --- a/tests/system-test/7-tmq/replay.py +++ b/tests/system-test/7-tmq/replay.py @@ -110,7 +110,7 @@ class TDTestCase: tdLog.info(shellCmd) os.system(shellCmd) - def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + def create_database(self,tsql, dbName,dropFlag=1,vgroups=1,replica=1): if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) @@ -149,21 +149,12 @@ class TDTestCase: t = time.time() startTs = int(round(t * 1000)) - #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) - rowsOfSql = 0 - for i in range(ctbNum): - sql += " %s_%d values "%(stbName,i) - for j in range(rowsPerTbl): - sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) - rowsOfSql += 1 - if ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): - tsql.execute(sql) - time.sleep(1) - rowsOfSql = 0 - if j < rowsPerTbl - 1: - sql = "insert into %s_%d values " %(stbName,i) - else: - sql = "insert into " + for j in range(rowsPerTbl): + for i in range(ctbNum): + sql += " %s_%d values (%d, %d, 'tmqrow_%d') "%(stbName, i, startTs + j + i, j+i, j+i) + tsql.execute(sql) + time.sleep(1) + sql = "insert into " #end sql if sql != pre_insert: #print("insert sql:%s"%sql) @@ -199,10 +190,10 @@ class TDTestCase: 'actionType': 0, \ 'dbName': 'db8', \ 'dropFlag': 1, \ - 'vgroups': 4, \ + 'vgroups': 1, \ 'replica': 1, \ 'stbName': 'stb1', \ - 'ctbNum': 1, \ + 'ctbNum': 2, \ 'rowsPerTbl': 10, \ 'batchNum': 1, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 @@ -223,7 +214,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] * 2 topicList = topicFromStb1 ifcheckdata = 0 ifManualCommit = 1 @@ -247,8 +238,8 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != 0: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") # tdLog.info("start consume 1 processor") diff --git a/tests/system-test/7-tmq/tmq_replay.py b/tests/system-test/7-tmq/tmq_replay.py new file mode 100644 index 0000000000..1e19d58516 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_replay.py @@ -0,0 +1,39 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + buildPath = tdCom.getBuildPath() + + cmdStr1 = '%s/build/bin/replay_test'%(buildPath) + tdLog.info(cmdStr1) + result = os.system(cmdStr1) + + if result != 0: + tdLog.exit("tmq_replay error!") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 343e3d8454..db5eb21ad8 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(get_db_name_test get_db_name_test.c) add_executable(tmq_offset tmqOffset.c) add_executable(tmq_offset_test tmq_offset_test.c) add_executable(varbinary_test varbinary_test.c) +add_executable(replay_test replay_test.c) if(${TD_LINUX}) add_executable(tsz_test tsz_test.c) @@ -57,6 +58,14 @@ target_link_libraries( PUBLIC os ) +target_link_libraries( + replay_test + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) + target_link_libraries( write_raw_block_test PUBLIC taos diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c new file mode 100644 index 0000000000..1fbaac0796 --- /dev/null +++ b/utils/test/c/replay_test.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "types.h" + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "g1"); + tmq_conf_set(conf, "client.id", "c1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.replay", "true"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +void test_vgroup_error(TAOS* pConn){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 2 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + ASSERT(tmq_subscribe(tmq, topic_list) != 0); + tmq_list_destroy(topic_list); + tmq_consumer_close(tmq); +} + +void test_stable_db_error(TAOS* pConn){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 1 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic t1 as stable d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + ASSERT(tmq_subscribe(tmq, topic_list) != 0); + tmq_list_destroy(topic_list); + tmq_consumer_close(tmq); + + pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic t1 as database d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + topic_list = tmq_list_new(); + tmq_list_append(topic_list, "t1"); + tmq = build_consumer(); + ASSERT(tmq_subscribe(tmq, topic_list) != 0); + tmq_list_destroy(topic_list); + tmq_consumer_close(tmq); +} + +void insert_with_sleep(TAOS* pConn, int32_t* interval, int32_t len){ + for(int i = 0; i < len; i++){ + TAOS_RES* pRes = taos_query(pConn, "insert into d1.table1 (ts, c1) values (now, 1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + taosMsleep(interval[i]); + } +} + +void insert_with_sleep_multi(TAOS* pConn, int32_t* interval, int32_t len){ + for(int i = 0; i < len; i++){ + TAOS_RES* pRes = taos_query(pConn, "insert into d1.table1 (ts, c1) values (now, 1) (now+1s, 2) d1.table2 (ts, c1) values (now, 1) (now+1s, 2)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + taosMsleep(interval[i]); + } +} + +void test_case1(TAOS* pConn, int32_t* interval, int32_t len){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 2 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table1 using d1.s1 tags(1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + insert_with_sleep(pConn, interval, len); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.table1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + tmq_list_destroy(topic_list); + + int32_t timeout = 5000; + + int64_t t = 0; + int32_t totalRows = 0; + char buf[1024] = {0}; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, timeout); + if (tmqmessage) { + if(t != 0){ + ASSERT(taosGetTimestampMs() - t >= interval[totalRows - 1]); + } + t = taosGetTimestampMs(); + + TAOS_ROW row = taos_fetch_row(tmqmessage); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); + int32_t numOfFields = taos_field_count(tmqmessage); + const char* tbName = tmq_get_table_name(tmqmessage); + taos_print_row(buf, row, fields, numOfFields); + + printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + totalRows++; + taos_free_result(tmqmessage); + } else { + break; + } + } + + ASSERT(totalRows == len); + tmq_consumer_close(tmq); +} + +void test_case2(TAOS* pConn, int32_t* interval, int32_t len, tsem_t* sem){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 1 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table1 using d1.s1 tags(1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table2 using d1.s1 tags(2)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + insert_with_sleep_multi(pConn, interval, len); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + tmq_list_destroy(topic_list); + + int32_t timeout = 5000; + + int64_t t = 0; + int32_t totalRows = 0; + char buf[1024] = {0}; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, timeout); + if (tmqmessage) { + if(t != 0 && totalRows % 4 == 0){ + ASSERT(taosGetTimestampMs() - t >= interval[totalRows/4 - 1]); + } + t = taosGetTimestampMs(); + + while(1){ + TAOS_ROW row = taos_fetch_row(tmqmessage); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); + int32_t numOfFields = taos_field_count(tmqmessage); + const char* tbName = tmq_get_table_name(tmqmessage); + taos_print_row(buf, row, fields, numOfFields); + + printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + totalRows++; + } + + taos_free_result(tmqmessage); + + if(totalRows == len * 4){ + taosSsleep(1); + tsem_post(sem); + } + } else { + break; + } + } + + ASSERT(totalRows == len * 4 + 1); + tmq_consumer_close(tmq); +} + +void* insertThreadFunc(void* param) { + tsem_t* sem = (tsem_t*)param; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + + tsem_wait(sem); + + TAOS_RES* pRes = taos_query(pConn, "insert into d1.table1 (ts, c1) values (now, 11)"); + ASSERT(taos_errno(pRes) == 0); + printf("insert data again\n"); + taos_free_result(pRes); + taos_close(pConn); + return NULL; +} + +int main(int argc, char* argv[]) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + test_vgroup_error(pConn); + test_stable_db_error(pConn); + + tsem_t sem; + tsem_init(&sem, 0, 0); + TdThread thread; + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + // pthread_create one thread to consume + taosThreadCreate(&thread, &thattr, insertThreadFunc, (void*)(&sem)); + + int32_t interval[5] = {1000, 200, 3000, 40, 500}; + test_case1(pConn, interval, sizeof(interval)/sizeof(int32_t)); + printf("test_case1 success\n"); + test_case2(pConn, interval, sizeof(interval)/sizeof(int32_t), &sem); + taos_close(pConn); + + taosThreadJoin(thread, NULL); + taosThreadClear(&thread); + tsem_destroy(&sem); + return 0; +} diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 6b774b3eff..34f4a9d094 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -621,10 +621,11 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn taos_print_row(buf, row, fields, numOfFields); if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); + taosFprintfFile(g_fp, "%lld tbname:%s, rows[%d]: %s\n", taosGetTimestampMs(), (tbName != NULL ? tbName : "null table"), totalRows, buf); // if (0 != g_stConfInfo.saveRowFlag) { // saveConsumeContentToTbl(pInfo, buf); // } +// taosFsyncFile(g_fp); } totalRows++; From 3e2e924e9867b41691a6b547485f626d9b590278 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 18:36:39 +0800 Subject: [PATCH 10/47] feat:[TD-26056] add replay logic --- source/dnode/mnode/impl/src/mndConsumer.c | 31 +++++++++++++++-------- source/dnode/vnode/src/tq/tqUtil.c | 6 ++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 05156e1427..c9ee66d3a0 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -126,31 +126,37 @@ void mndRebCntDec() { } static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { - int32_t numOfTopics = taosArrayGetSize(pTopicList); + SMqTopicObj *pTopic = NULL; + int32_t code = 0; + int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { char *pOneTopic = taosArrayGetP(pTopicList, i); - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOneTopic); + pTopic = mndAcquireTopic(pMnode, pOneTopic); if (pTopic == NULL) { // terrno has been set by callee function - return -1; + code = -1; + goto FAILED; } if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) { - mndReleaseTopic(pMnode, pTopic); - return -1; + code = -1; + goto FAILED; } if(enableReplay){ if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){ - return TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; + code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; + goto FAILED; }else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); if (pDb == NULL) { - mndReleaseTopic(pMnode, pTopic); - return -1; + code = -1; + goto FAILED; } if (pDb->cfg.numOfVgroups != 1) { - return TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; + mndReleaseDb(pMnode, pDb); + code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; + goto FAILED; } mndReleaseDb(pMnode, pDb); } @@ -158,13 +164,16 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * mndTransSetDbName(pTrans, pOneTopic, NULL); if(mndTransCheckConflict(pMnode, pTrans) != 0){ - mndReleaseTopic(pMnode, pTopic); - return -1; + code = -1; + goto FAILED; } mndReleaseTopic(pMnode, pTopic); } return 0; +FAILED: + mndReleaseTopic(pMnode, pTopic); + return code; } static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 215f8d3cb2..a4c3d395e3 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -40,11 +40,11 @@ void tqUpdateNodeStage(STQ* pTq) { tqDebug("vgId:%d update the meta stage to be:%"PRId64, pTq->pStreamMeta->vgId, pTq->pStreamMeta->stage); } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset, bool withTbName) { +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { pRsp->reqOffset = pOffset; pRsp->rspOffset = pOffset; - pRsp->withTbName = withTbName; + pRsp->withTbName = 1; pRsp->withSchema = 1; pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -177,7 +177,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t vgId = TD_VID(pTq->pVnode); SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, *offset, pRequest->withTbName); + tqInitTaosxRsp(&taosxRsp, *offset); if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { From d10915dce6a3cced5024c307834650f7ee19191e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 19:03:13 +0800 Subject: [PATCH 11/47] feat:[TD-26056] add replay logic --- utils/test/c/replay_test.c | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c index 1fbaac0796..df493ce2ac 100644 --- a/utils/test/c/replay_test.c +++ b/utils/test/c/replay_test.c @@ -281,6 +281,93 @@ void test_case2(TAOS* pConn, int32_t* interval, int32_t len, tsem_t* sem){ tmq_consumer_close(tmq); } +void test_case3(TAOS* pConn, int32_t* interval, int32_t len){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists t1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists d1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists d1 vgroups 1 wal_retention_period 3600"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "CREATE STABLE d1.s1 (ts TIMESTAMP, c1 INT) TAGS (t1 INT)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table1 using d1.s1 tags(1)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1.table2 using d1.s1 tags(2)"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + insert_with_sleep_multi(pConn, interval, len); + + pRes = taos_query(pConn, "create topic t1 as select * from d1.s1"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + tmq_list_t* topic_list = tmq_list_new(); + + tmq_list_append(topic_list, "t1"); + tmq_t* tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + + int32_t timeout = 5000; + + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, timeout); + taos_free_result(tmqmessage); + + tmq_consumer_close(tmq); + + tmq = build_consumer(); + // 启动订阅 + tmq_subscribe(tmq, topic_list); + + int64_t t = 0; + int32_t totalRows = 0; + char buf[1024] = {0}; + while (1) { + tmqmessage = tmq_consumer_poll(tmq, timeout); + if (tmqmessage) { + if(t != 0 && totalRows % 4 == 0){ + ASSERT(taosGetTimestampMs() - t >= interval[totalRows/4 - 1]); + } + t = taosGetTimestampMs(); + + while(1){ + TAOS_ROW row = taos_fetch_row(tmqmessage); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); + int32_t numOfFields = taos_field_count(tmqmessage); + const char* tbName = tmq_get_table_name(tmqmessage); + taos_print_row(buf, row, fields, numOfFields); + + printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + totalRows++; + } + + taos_free_result(tmqmessage); + } else { + break; + } + } + + ASSERT(totalRows == len * 4); + + tmq_consumer_close(tmq); + tmq_list_destroy(topic_list); +} + void* insertThreadFunc(void* param) { tsem_t* sem = (tsem_t*)param; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -314,6 +401,8 @@ int main(int argc, char* argv[]) { test_case1(pConn, interval, sizeof(interval)/sizeof(int32_t)); printf("test_case1 success\n"); test_case2(pConn, interval, sizeof(interval)/sizeof(int32_t), &sem); + printf("test_case2 success\n"); + test_case3(pConn, interval, sizeof(interval)/sizeof(int32_t)); taos_close(pConn); taosThreadJoin(thread, NULL); From b12c734f096b348143a475969c138ee841b9c2a9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 Oct 2023 21:14:32 +0800 Subject: [PATCH 12/47] feat:[TD-26056] add replay logic --- utils/test/c/tmqSim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 34f4a9d094..14e30008fe 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -621,7 +621,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn taos_print_row(buf, row, fields, numOfFields); if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "%lld tbname:%s, rows[%d]: %s\n", taosGetTimestampMs(), (tbName != NULL ? tbName : "null table"), totalRows, buf); + taosFprintfFile(g_fp, "time:%" PRId64 " tbname:%s, rows[%d]: %s\n", taosGetTimestampMs(), (tbName != NULL ? tbName : "null table"), totalRows, buf); // if (0 != g_stConfInfo.saveRowFlag) { // saveConsumeContentToTbl(pInfo, buf); // } From b8fe279a855edb5edabfe5608a970d19d42caf65 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 07:44:29 +0800 Subject: [PATCH 13/47] fix: renanme function --- source/libs/parser/src/parTranslater.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4e0979ddaa..201111e425 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1953,7 +1953,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_SUCCESS; } -static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1976,7 +1976,7 @@ static int32_t translateWindowPseudoColumnFunc2(STranslateContext* pCxt, SNode** return TSDB_CODE_SUCCESS; } -static int32_t translateScanPseudoColumnFunc2(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -2008,7 +2008,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { - code = translateScanPseudoColumnFunc2(pCxt, ppNode); + code = translateScanPseudoColumnFunc(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateIndefiniteRowsFunc(pCxt, pFunc); @@ -2017,7 +2017,7 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateForbidFillFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindowPseudoColumnFunc2(pCxt, ppNode); + code = translateWindowPseudoColumnFunc(pCxt, ppNode); } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); From da9d39bfe6ec09afa701325f975670421b5e2d12 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 08:31:23 +0800 Subject: [PATCH 14/47] fix: cancel future pseudo column translation function processing when rewritten to column --- source/libs/parser/src/parTranslater.c | 27 ++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 201111e425..e285c7a806 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1621,7 +1621,7 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } -static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1645,6 +1645,7 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** } } if (!bFound) { + *pRewriteToColumn = true; int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1953,7 +1954,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_SUCCESS; } -static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1966,6 +1967,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** pFunc->functionName); } if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { + *pRewriteToColumn = true; int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1976,7 +1978,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** return TSDB_CODE_SUCCESS; } -static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode) { +static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1986,6 +1988,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + *pRewriteToColumn = true; int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2008,7 +2011,11 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { - code = translateScanPseudoColumnFunc(pCxt, ppNode); + bool bRewriteToColumn = false; + code = translateScanPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); + if (bRewriteToColumn) { + return code; + } } if (TSDB_CODE_SUCCESS == code) { code = translateIndefiniteRowsFunc(pCxt, pFunc); @@ -2017,7 +2024,11 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateForbidFillFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindowPseudoColumnFunc(pCxt, ppNode); + bool bRewriteToColumn = false; + code = translateWindowPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); + if (bRewriteToColumn) { + return code; + } } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); @@ -2038,7 +2049,11 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) code = translateInterpFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { - code = translateInterpPseudoColumnFunc(pCxt, ppNode); + bool bRewriteToColumn = false; + code = translateInterpPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); + if (bRewriteToColumn) { + return code; + } } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); From d42e819d2d89d00a43e10c32ba0a92f64821bffd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 09:02:37 +0800 Subject: [PATCH 15/47] feat:[TD-26056] add replay logic --- utils/test/c/replay_test.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c index df493ce2ac..c105670733 100644 --- a/utils/test/c/replay_test.c +++ b/utils/test/c/replay_test.c @@ -184,10 +184,9 @@ void test_case1(TAOS* pConn, int32_t* interval, int32_t len){ TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); int32_t numOfFields = taos_field_count(tmqmessage); - const char* tbName = tmq_get_table_name(tmqmessage); taos_print_row(buf, row, fields, numOfFields); - printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + printf("time:%" PRId64 " rows[%d]: %s\n", t, totalRows, buf); totalRows++; taos_free_result(tmqmessage); } else { @@ -259,10 +258,9 @@ void test_case2(TAOS* pConn, int32_t* interval, int32_t len, tsem_t* sem){ TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); int32_t numOfFields = taos_field_count(tmqmessage); - const char* tbName = tmq_get_table_name(tmqmessage); taos_print_row(buf, row, fields, numOfFields); - printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + printf("time:%" PRId64 " rows[%d]: %s\n", t, totalRows, buf); totalRows++; } @@ -349,10 +347,9 @@ void test_case3(TAOS* pConn, int32_t* interval, int32_t len){ TAOS_FIELD* fields = taos_fetch_fields(tmqmessage); int32_t numOfFields = taos_field_count(tmqmessage); - const char* tbName = tmq_get_table_name(tmqmessage); taos_print_row(buf, row, fields, numOfFields); - printf("%lld tbname:%s, rows[%d]: %s\n", t, (tbName != NULL ? tbName : "null table"), totalRows, buf); + printf("time:%" PRId64 " rows[%d]: %s\n", t, totalRows, buf); totalRows++; } From bc2cf345fade4301cba25785e98817d5cd0751fb Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 10:44:13 +0800 Subject: [PATCH 16/47] fix: change error code after replace column and translate --- source/libs/parser/src/parTranslater.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e285c7a806..c1881e3d27 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1973,7 +1973,11 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** return code; } translateColumn(pCxt, (SColumnNode**)ppNode); - return pCxt->errCode; + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); + } else { + return TSDB_CODE_SUCCESS; + } } return TSDB_CODE_SUCCESS; } @@ -1994,7 +1998,11 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp return code; } translateColumn(pCxt, (SColumnNode**)ppNode); - return pCxt->errCode; + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } else { + return TSDB_CODE_SUCCESS; + } } } else { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); From 53c84a7245c952514c485e227f2827c96230df65 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 11:02:01 +0800 Subject: [PATCH 17/47] doc:add doc for replay --- docs/en/07-develop/07-tmq.mdx | 18 ++++++++++++++++++ docs/zh/07-develop/07-tmq.mdx | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index f833dbf439..a511eaa4c3 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -514,6 +514,24 @@ var consumer = new ConsumerBuilder(cfg).Build(); A consumer group is automatically created when multiple consumers are configured with the same consumer group ID. +Data replay function description: +- Subscription adds replay function, which replays according to the time of data writing. + For example, writing three pieces of data at the following time. + ```sql + 2023/09/22 00:00:00.000 + 2023/09/22 00:00:05.000 + 2023/09/22 00:00:08.000 + ``` + After subscribing to the first data for 5 seconds, the second data is returned, and after obtaining the second data for 3 seconds, the third data is returned. +- Only column subscriptions support data replay. + - Replay needs to ensure an independent timeline + - If it is a sub table subscription or a normal table subscription, only one vnode has data, ensuring a timeline. + - If subscribing to a super table, it is necessary to ensure that the DB has only one vnode, otherwise an error will be reported (because the data subscribed to on multiple vnodes is not on the same timeline). +- Super table and database subscriptions do not support replay +- Add the enable.replay parameter. True indicates that the subscription replay function is enabled, while false indicates that the subscription replay function is not enabled by default. +- Replay does not support progress saving, so when the replay parameter enable, auto commit will automatically close. +- Due to the processing time required for data replay, there is an error of tens of milliseconds in the accuracy of replay. + ## Subscribe to a Topic A single consumer can subscribe to multiple topics. diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 927d762829..fee0ec86c2 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -355,6 +355,7 @@ CREATE TOPIC topic_name [with meta] AS DATABASE db_name; | `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | | `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | | `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句） |默认关闭 | +| `enable.replay` | boolean | 是否开启数据回放功能 |默认关闭 | 对于不同编程语言，其设置方式如下： @@ -515,6 +516,24 @@ var consumer = new ConsumerBuilder(cfg).Build(); 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。 +数据回放功能说明： +- 订阅增加 replay 功能，按照数据写入的时间回放。 + 比如，如下时间写入三条数据 + ```sql + 2023/09/22 00:00:00.000 + 2023/09/22 00:00:05.000 + 2023/09/22 00:00:08.000 + ``` + 则订阅出第一条数据 5s 后返回第二条数据，获取第二条数据 3s 后返回第三条数据。 +- 仅列订阅支持数据回放 + - 回放需要保证独立时间线 + - 如果是子表订阅或者普通表订阅，只有一个vnode上有数据，保证是一个时间线 + - 如果超级表订阅，则需保证该 DB 只有一个vnode，否则报错（因为多个vnode上订阅出的数据不在一个时间线上） +- 超级表和库订阅不支持回放 +- 增加 enable.replay 参数，true表示开启订阅回放功能，false表示不开启订阅回放功能，默认不开启。 +- 回放不支持进度保存，所以回放参数 enable.replay = true 时，auto commit 自动关闭 +- 因为数据回放本身需要处理时间，所以回放的精度存在几十ms的误差 + ## 订阅 *topics* 一个 consumer 支持同时订阅多个 topic。 From 9113c3c3b6032e9d2d0c2d4c7e4592f80da2b672 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 11:53:54 +0800 Subject: [PATCH 18/47] fix:rollback removed code --- docs/en/07-develop/07-tmq.mdx | 1 + source/dnode/vnode/src/tq/tqRead.c | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index a511eaa4c3..86b2498345 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -356,6 +356,7 @@ You configure the following parameters when creating a consumer: | `enable.auto.commit` | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself | Default value is true | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | | `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | default value: false +| `enable.replay` | boolean | Specify whether data replay function enabled or not |default value: false | The method of specifying these parameters depends on the language used: diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 65914cc70e..3b052a3edd 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -788,6 +788,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas int32_t sversion = pSubmitTbData->sver; int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); From 1762a85127d8a16d6b34b51a4e64ca31981d421d Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 10 Oct 2023 14:33:53 +0800 Subject: [PATCH 19/47] enhance: add test case --- tests/develop-test/2-query/pseudo_column.py | 81 +++++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 82 insertions(+) create mode 100644 tests/develop-test/2-query/pseudo_column.py diff --git a/tests/develop-test/2-query/pseudo_column.py b/tests/develop-test/2-query/pseudo_column.py new file mode 100644 index 0000000000..9f6366a3c6 --- /dev/null +++ b/tests/develop-test/2-query/pseudo_column.py @@ -0,0 +1,81 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TS-3904/TS-3005] pseudo column test case + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self._conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use pseudo_col") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists pseudo_col") + tdSql.execute("create database if not exists pseudo_col") + tdSql.execute('use pseudo_col') + tdSql.execute('create table st(ts timestamp, f int) tags (t int)') + tdSql.execute("insert into ct1 using st tags(1) values('2023-10-10 14:10:00', 1)('2023-10-10 14:10:01', 11)") + tdSql.execute("insert into ct2 using st tags(2) values('2023-10-10 14:10:02', 2)('2023-10-10 14:10:03', 22)") + + tdSql.query('select tbname from (select tbname from st) order by tbname') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + + tdSql.query('select `tbname` from (select tbname from st) order by tbname') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + + tdSql.query('select `st.tbname` from (select st.tbname from st) order by `st.tbname`') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + + tdSql.error('select tbname from (select * from st)') + tdSql.error('select st.tbname from (select st.tbname from st)') + tdSql.error('select `st.tbname` from (select st.tbname from st) order by tbname') + + tdSql.query('select _wstart, _wend, _wduration, c from (select _wstart, _wend, _wduration, count(*) as c from st interval(1s)) order by _wstart') + tdSql.checkCols(4) + tdSql.checkRows(4) + tdSql.checkData(0, 1, '2023-10-10 14:10:01') + tdSql.checkData(0, 3, 1) + + tdSql.error('select _wstart, _wend, _wduration, c from (select count(*) as c from st) order by _wstart') + + tdSql.query("select _irowts, if2 from (select _irowts, interp(f) as if2 from st range('2023-10-10 14:10:00', '2023-10-10 14:10:10') every(1s) fill(value, 8))") + tdSql.checkRows(11) + tdSql.checkData(9, 1, 8); + tdSql.execute('drop database pseudo_col') + + tdSql.error("select _irowts, if2 from (select interp(f) as if2 from st range('2023-10-10 14:10:00', '2023-10-10 14:10:10') every(1s) fill(value, 8))") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 36b8fded81..7afef03f0d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1263,6 +1263,7 @@ #develop test ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py +,,n,develop-test,python3 ./test.py -f 2-query/pseudo_column.py ,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py From 440fa772828229b99d21851e067d827a30f91381 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Oct 2023 16:55:36 +0800 Subject: [PATCH 20/47] fix:merge datablock if data in same wal version --- source/dnode/vnode/src/tq/tqRead.c | 82 ++++++++++-------------------- 1 file changed, 28 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 3b052a3edd..1c2561b1a8 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -366,82 +366,56 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con // todo ignore the error in wal? bool tqNextBlockInWal(STqReader* pReader, const char* id) { SWalReader* pWalReader = pReader->pWalReader; + SSDataBlock* pDataBlock = NULL; uint64_t st = taosGetTimestampMs(); while (1) { - SArray* pBlockList = pReader->submit.aSubmitTbData; - if (pBlockList == NULL || pReader->nextBlk >= taosArrayGetSize(pBlockList)) { - // try next message in wal file - // todo always retry to avoid read failure caused by wal file deletion - if (walNextValidMsg(pWalReader) < 0) { - return false; - } - - void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); - int64_t ver = pWalReader->pHead->head.version; - - SDecoder decoder = {0}; - tDecoderInit(&decoder, pBody, bodyLen); - - { - int32_t nSubmitTbData = taosArrayGetSize(pReader->submit.aSubmitTbData); - for (int32_t i = 0; i < nSubmitTbData; i++) { - SSubmitTbData* pData = taosArrayGet(pReader->submit.aSubmitTbData, i); - if (pData->pCreateTbReq != NULL) { - taosArrayDestroy(pData->pCreateTbReq->ctb.tagName); - taosMemoryFreeClear(pData->pCreateTbReq); - } - pData->aRowP = taosArrayDestroy(pData->aRowP); - } - pReader->submit.aSubmitTbData = taosArrayDestroy(pReader->submit.aSubmitTbData); - } - - if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { - tDecoderClear(&decoder); - tqError("decode wal file error, msgLen:%d, ver:%" PRId64, bodyLen, ver); - return false; - } - - tDecoderClear(&decoder); - pReader->nextBlk = 0; + // try next message in wal file + if (walNextValidMsg(pWalReader) < 0) { + return false; } + void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pWalReader->pHead->head.version; + + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); + pReader->nextBlk = 0; int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqTrace("tq reader next data block %d/%d, len:%d %" PRId64 " %d", pReader->nextBlk, - numOfBlocks, pReader->msg.msgLen, pReader->msg.ver, pReader->nextBlk); + tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, + numOfBlocks, pReader->msg.msgLen, pReader->msg.ver); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if (pReader->tbIdHash == NULL) { - SSDataBlock* pRes = NULL; - int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); - if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - return true; - } - } - - void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); - if (ret != NULL) { - tqTrace("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); - + if (pReader->tbIdHash == NULL || taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)) != NULL) { + tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid); SSDataBlock* pRes = NULL; int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - return true; + if(pDataBlock == NULL){ + pDataBlock = createOneDataBlock(pRes, true); + }else{ + blockDataMerge(pDataBlock, pRes); + } } } else { pReader->nextBlk += 1; tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); } } - - qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->msg.msgStr = NULL; + if(pDataBlock != NULL){ + blockDataCleanup(pReader->pResBlock); + copyDataBlock(pReader->pResBlock, pDataBlock); + blockDataDestroy(pDataBlock); + return true; + }else{ + qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); + } + if(taosGetTimestampMs() - st > 1000){ return false; } From e9fc079d269133f9e05dad8a60c04b2ed6c25e3e Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Tue, 10 Oct 2023 14:28:35 +0800 Subject: [PATCH 21/47] opt bloom filter --- include/util/tbloomfilter.h | 6 +++- include/util/tscalablebf.h | 3 ++ source/libs/stream/src/streamUpdate.c | 13 +++++--- source/util/src/tbloomfilter.c | 44 ++++++++++++++++----------- source/util/src/tscalablebf.c | 28 +++++++++++++++-- source/util/test/bloomFilterTest.cpp | 8 +++-- 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index 16f7ff7958..23bf5aaafb 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -24,6 +24,9 @@ extern "C" { #endif +#define HASH_FUNCTION_1 taosFastHash +#define HASH_FUNCTION_2 taosDJB2Hash + typedef struct SBloomFilter { uint32_t hashFunctions; uint64_t expectedEntries; @@ -37,8 +40,9 @@ typedef struct SBloomFilter { } SBloomFilter; SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); +int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, uint32_t len); +int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); void tBloomFilterDestroy(SBloomFilter *pBF); void tBloomFilterDump(const SBloomFilter *pBF); bool tBloomFilterIsFull(const SBloomFilter *pBF); diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 9977c1436d..2cf170cf04 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -26,9 +26,12 @@ typedef struct SScalableBf { SArray *bfArray; // array of bloom filters uint32_t growth; uint64_t numBits; + _hash_fn_t hashFn1; + _hash_fn_t hashFn2; } SScalableBf; SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); +int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); void tScalableBfDestroy(SScalableBf *pSBf); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index f9ab672c4b..a463a17ec8 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -218,17 +218,22 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } SScalableBf *pSBf = getSBf(pInfo, ts); - // pSBf may be a null pointer - if (pSBf) { - res = tScalableBfPut(pSBf, &updateKey, sizeof(SUpdateKey)); - } int32_t size = taosHashGetSize(pInfo->pMap); if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && *pMapMaxTs < ts)) { taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); + // pSBf may be a null pointer + if (pSBf) { + res = tScalableBfPutNoCheck(pSBf, &updateKey, sizeof(SUpdateKey)); + } return false; } + // pSBf may be a null pointer + if (pSBf) { + res = tScalableBfPut(pSBf, &updateKey, sizeof(SUpdateKey)); + } + if (!pMapMaxTs && maxTs < ts) { taosArraySet(pInfo->pTsBuckets, index, &ts); return false; diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 84a78f3477..150faab571 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -24,9 +24,8 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; - buf[unitIndex] |= mask; + buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS)); return buf[unitIndex] != old; } @@ -57,10 +56,8 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { // ln(2) = 0.693147180559945 pBF->hashFunctions = (uint32_t)ceil(lnRate / 0.693147180559945); - /*pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);*/ - /*pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR);*/ - pBF->hashFn1 = taosFastHash; - pBF->hashFn2 = taosDJB2Hash; + pBF->hashFn1 = HASH_FUNCTION_1; + pBF->hashFn2 = HASH_FUNCTION_2; pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); @@ -69,14 +66,29 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) { return pBF; } -int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { +int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { ASSERT(!tBloomFilterIsFull(pBF)); + bool hasChange = false; + const register uint64_t size = pBF->numBits; + uint64_t cbHash = hash1; + for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { + hasChange |= setBit(pBF->buffer, cbHash % size); + cbHash += hash2; + } + if (hasChange) { + pBF->size++; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_FAILED; +} + +int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); bool hasChange = false; const register uint64_t size = pBF->numBits; uint64_t cbHash = h1; - for (uint64_t i = 0; i < pBF->hashFunctions; ++i) { + for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { hasChange |= setBit(pBF->buffer, cbHash % size); cbHash += h2; } @@ -87,16 +99,14 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) { return TSDB_CODE_FAILED; } -int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, uint32_t len) { - uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len); - uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len); +int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) { const register uint64_t size = pBF->numBits; - uint64_t cbHash = h1; - for (uint64_t i = 0; i < pBF->hashFunctions; ++i) { + uint64_t cbHash = hash1; + for (uint32_t i = 0; i < pBF->hashFunctions; ++i) { if (!getBit(pBF->buffer, cbHash % size)) { return TSDB_CODE_SUCCESS; } - cbHash += h2; + cbHash += hash2; } return TSDB_CODE_FAILED; } @@ -137,10 +147,8 @@ SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) { if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; } if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; - /*pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);*/ - /*pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR);*/ - pBF->hashFn1 = taosFastHash; - pBF->hashFn2 = taosDJB2Hash; + pBF->hashFn1 = HASH_FUNCTION_1; + pBF->hashFn2 = HASH_FUNCTION_2; return pBF; _error: diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 797f3a924d..3b4975b701 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -39,13 +39,31 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { return NULL; } pSBf->growth = DEFAULT_GROWTH; + pSBf->hashFn1 = HASH_FUNCTION_1; + pSBf->hashFn2 = HASH_FUNCTION_2; return pSBf; } +int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + int32_t size = taosArrayGetSize(pSBf->bfArray); + SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); + ASSERT(pNormalBf); + if (tBloomFilterIsFull(pNormalBf)) { + pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, + pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); + if (pNormalBf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return tBloomFilterPut(pNormalBf, keyBuf, len); +} + int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); + uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { - if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), keyBuf, len) != TSDB_CODE_SUCCESS) { + if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } } @@ -59,13 +77,15 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { return TSDB_CODE_OUT_OF_MEMORY; } } - return tBloomFilterPut(pNormalBf, keyBuf, len); + return tBloomFilterPutHash(pNormalBf, h1, h2); } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); + uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 1; i >= 0; --i) { - if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), keyBuf, len) != TSDB_CODE_SUCCESS) { + if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } } @@ -113,6 +133,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + pSBf->hashFn1 = HASH_FUNCTION_1; + pSBf->hashFn2 = HASH_FUNCTION_2; pSBf->bfArray = NULL; int32_t size = 0; if (tDecodeI32(pDecoder, &size) < 0) goto _error; diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 2e02129164..c51de3c8a4 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -43,12 +43,16 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_FAILED); + uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED); } for (int64_t i = 2000; i < 3000; i++) { int64_t ts = i + ts1; - GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); + uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t)); + uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t)); + GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS); } tBloomFilterDestroy(pBF1); From 71a3e3ce169290af7f676ab45282cef68431f257 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 11 Oct 2023 10:29:41 +0800 Subject: [PATCH 22/47] ci --- tests/script/tsim/stream/checkpointSession1.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/stream/checkpointSession1.sim b/tests/script/tsim/stream/checkpointSession1.sim index 5c9625aabb..e0ba08f9e7 100644 --- a/tests/script/tsim/stream/checkpointSession1.sim +++ b/tests/script/tsim/stream/checkpointSession1.sim @@ -63,7 +63,7 @@ sql insert into t2 values(1648791233003,4,2,3,1.1); $loop_count = 0 loop1: -sleep 1000 +sleep 2000 sql select * from streamt; From 8362759e471d32870ef74907209bb8bd241a94c1 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 11 Oct 2023 13:46:11 +0800 Subject: [PATCH 23/47] ci --- tests/script/tsim/stream/checkpointSession1.sim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/stream/checkpointSession1.sim b/tests/script/tsim/stream/checkpointSession1.sim index e0ba08f9e7..ea93e1525a 100644 --- a/tests/script/tsim/stream/checkpointSession1.sim +++ b/tests/script/tsim/stream/checkpointSession1.sim @@ -57,13 +57,15 @@ system sh/stop_dnodes.sh system sh/exec.sh -n dnode1 -s start +sleep 2000 + sql insert into t1 values(1648791213002,3,2,3,1.1); sql insert into t2 values(1648791233003,4,2,3,1.1); $loop_count = 0 loop1: -sleep 2000 +sleep 1000 sql select * from streamt; From 5fafdf351b0cde9080338514ddac15014708b6d5 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 11 Oct 2023 20:17:47 +0800 Subject: [PATCH 24/47] refect --- source/libs/executor/src/streamtimewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9da3c99e7..6fc862b438 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3590,7 +3590,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) { for (int32_t i = 0; i < num; i++) { SStateWindowInfo curInfo = {0}; SStateWindowInfo nextInfo = {0}; - SStateWindowInfo dummy = {0}; qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo); From 35daa51878b8ceb1ed5c244120c237ec4b116765 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 11 Oct 2023 20:42:52 +0800 Subject: [PATCH 25/47] file path contains spaces on linux/mac --- source/client/src/clientEnv.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index dbddf9cac6..30f39d2b89 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -718,6 +718,22 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { +#ifndef WINDOWS + int len = strlen(str); + if (len > 1 && str[0] != '"' && str[0] != '\'') { + if (len + 2 >= PATH_MAX) { + tscError("Too long path %s", str); + return -1; + } + char newstr[PATH_MAX]; + newstr[0] = '"'; + strncpy(newstr+1, str, len); + newstr[len + 1] = '"'; + newstr[len + 2] = '\0'; + str = newstr; + } +#endif + if (option == TSDB_OPTION_CONFIGDIR) { tstrncpy(configDir, str, PATH_MAX); tscInfo("set cfg:%s to %s", configDir, str); From 8d1e427861ca6a8f63af2402f8819bbf100f4553 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 12 Oct 2023 10:56:03 +0800 Subject: [PATCH 26/47] fix change scope --- source/client/src/clientEnv.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 30f39d2b89..faf72108aa 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -718,23 +718,22 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { -#ifndef WINDOWS - int len = strlen(str); - if (len > 1 && str[0] != '"' && str[0] != '\'') { - if (len + 2 >= PATH_MAX) { - tscError("Too long path %s", str); - return -1; - } - char newstr[PATH_MAX]; - newstr[0] = '"'; - strncpy(newstr+1, str, len); - newstr[len + 1] = '"'; - newstr[len + 2] = '\0'; - str = newstr; - } -#endif - if (option == TSDB_OPTION_CONFIGDIR) { +#ifndef WINDOWS + int len = strlen(str); + if (len > 1 && str[0] != '"' && str[0] != '\'') { + if (len + 2 >= PATH_MAX) { + tscError("Too long path %s", str); + return -1; + } + char newstr[PATH_MAX]; + newstr[0] = '"'; + strncpy(newstr+1, str, len); + newstr[len + 1] = '"'; + newstr[len + 2] = '\0'; + str = newstr; + } +#endif tstrncpy(configDir, str, PATH_MAX); tscInfo("set cfg:%s to %s", configDir, str); return 0; From 81dc3e7ea2110c90cf56c2210e49e0dd40350ce8 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 12 Oct 2023 15:04:53 +0800 Subject: [PATCH 27/47] fix test failed --- source/client/src/clientEnv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index faf72108aa..98782f74aa 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -720,13 +720,13 @@ int taos_init() { int taos_options_imp(TSDB_OPTION option, const char *str) { if (option == TSDB_OPTION_CONFIGDIR) { #ifndef WINDOWS + char newstr[PATH_MAX]; int len = strlen(str); if (len > 1 && str[0] != '"' && str[0] != '\'') { if (len + 2 >= PATH_MAX) { tscError("Too long path %s", str); return -1; } - char newstr[PATH_MAX]; newstr[0] = '"'; strncpy(newstr+1, str, len); newstr[len + 1] = '"'; From 1e1ed541fa83d900189e4c20b198c83eaf3e336c Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 13 Oct 2023 08:11:15 +0800 Subject: [PATCH 28/47] fix: add more test case --- tests/develop-test/2-query/pseudo_column.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/develop-test/2-query/pseudo_column.py b/tests/develop-test/2-query/pseudo_column.py index 9f6366a3c6..4a97be6612 100644 --- a/tests/develop-test/2-query/pseudo_column.py +++ b/tests/develop-test/2-query/pseudo_column.py @@ -47,6 +47,14 @@ class TDTestCase: tdSql.checkData(2, 0, 'ct2') tdSql.checkData(2, 0, 'ct2') + tdSql.query('select `tbname` from (select tbname from st) order by tbname') + tdSql.checkCols(1) + tdSql.checkRows(4) + tdSql.checkData(0, 0, 'ct1') + tdSql.checkData(1, 0, 'ct1') + tdSql.checkData(2, 0, 'ct2') + tdSql.checkData(2, 0, 'ct2') + tdSql.query('select `st.tbname` from (select st.tbname from st) order by `st.tbname`') tdSql.checkCols(1) tdSql.checkRows(4) From 429125be933cb6297680ec3fc07a25d50cf460d8 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 13 Oct 2023 11:32:35 +0800 Subject: [PATCH 29/47] session state recover --- include/libs/stream/tstreamFileState.h | 3 +++ .../executor/src/streamtimewindowoperator.c | 1 - source/libs/stream/src/tstreamFileState.c | 26 ++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 2b567a7370..c1974df7de 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -79,6 +79,9 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen); int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); +SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); +int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); + void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateCleanup(void* pBuff); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9da3c99e7..6fc862b438 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3590,7 +3590,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) { for (int32_t i = 0; i < num; i++) { SStateWindowInfo curInfo = {0}; SStateWindowInfo nextInfo = {0}; - SStateWindowInfo dummy = {0}; qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index ac404893f0..584e81fafc 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -177,7 +177,10 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ // todo(liuyao) optimize if (type == STREAM_STATE_BUFF_HASH) { recoverSnapshot(pFileState, checkpointId); + } else { + recoverSesssion(pFileState, checkpointId); } + return pFileState; _error: @@ -642,12 +645,24 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { } int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { - int code = TSDB_CODE_SUCCESS; + int code = TSDB_CODE_SUCCESS; + if (pFileState->maxTs != INT64_MIN) { + int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) + ? INT64_MIN + : pFileState->maxTs - pFileState->deleteMark; + deleteExpiredCheckPoint(pFileState, mark); + } + SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore); if (pCur == NULL) { return -1; } + int32_t recoverNum = TMIN(MIN_NUM_OF_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { + if (pFileState->curRowCount >= recoverNum) { + break; + } + void* pVal = NULL; int32_t vlen = 0; SSessionKey key = {0}; @@ -655,12 +670,14 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { if (code != 0) { break; } - taosMemoryFree(pVal); + SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); + putSessionWinResultBuff(pFileState, pPos); code = streamStateSessionCurPrev_rocksdb(pCur); } streamStateFreeCur(pCur); return code; } + int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { @@ -674,11 +691,12 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { if (pCur == NULL) { return -1; } - + int32_t recoverNum = TMIN(MIN_NUM_OF_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { - if (pFileState->curRowCount == pFileState->maxRowCount) { + if (pFileState->curRowCount >= recoverNum) { break; } + void* pVal = NULL; int32_t vlen = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); From d30c6f6bcc1e6a1102c57178a389b47b2976693c Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 16 Oct 2023 13:56:15 +0800 Subject: [PATCH 30/47] fix: change tablename.tbname aliasname to tbname --- source/libs/parser/src/parAstCreater.c | 8 ++- source/libs/parser/src/parTranslater.c | 57 +++++++++++++-------- tests/develop-test/2-query/pseudo_column.py | 10 +++- 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a418dc63b4..12062e0d4a 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -276,11 +276,9 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName); strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else if (pRawExpr->isPseudoColumn) { - int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->userAlias, pRawExpr->p, len); - pExpr->userAlias[len] = '\0'; - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + // all pseudo column are translate to function with same name + strcpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName); + strcpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 090d83d88f..42ab2f7a2f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1942,6 +1942,22 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_OUT_OF_MEMORY; } SExprNode* pOldExpr = (SExprNode*)(*ppNode); + //rewrite a.tbname == tbname(a) + if (nodeType(*ppNode) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(*ppNode))->funcType == FUNCTION_TYPE_TBNAME) { + SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); + if (0 != LIST_LENGTH(pFunc->pParameterList)) { + SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + pCol->node.resType = pOldExpr->resType; + strcpy(pCol->tableAlias, pVal->literal); + strcpy(pCol->colName, pFunc->functionName); + strcpy(pCol->node.aliasName, pCol->colName); + strcpy(pCol->node.userAlias, pCol->colName); + nodesDestroyNode(*ppNode); + *ppNode = (SNode*)pCol; + + return TSDB_CODE_SUCCESS; + } + } pCol->node.resType = pOldExpr->resType; strcpy(pCol->node.aliasName, pOldExpr->aliasName); strcpy(pCol->node.userAlias, pOldExpr->userAlias); @@ -1953,6 +1969,19 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_SUCCESS; } +static int32_t rewriteToColumnAndRetranslate(STranslateContext* pCxt, SNode** ppNode, int32_t errCode) { + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, errCode); + } else { + return TSDB_CODE_SUCCESS; + } +} + static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { @@ -1967,16 +1996,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** } if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { *pRewriteToColumn = true; - int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - translateColumn(pCxt, (SColumnNode**)ppNode); - if (pCxt->errCode != TSDB_CODE_SUCCESS) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); - } else { - return TSDB_CODE_SUCCESS; - } + return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_WINDOW_PC); } return TSDB_CODE_SUCCESS; } @@ -1992,23 +2012,18 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp } if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { *pRewriteToColumn = true; - int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - translateColumn(pCxt, (SColumnNode**)ppNode); - if (pCxt->errCode != TSDB_CODE_SUCCESS) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } else { - return TSDB_CODE_SUCCESS; - } + return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_TBNAME); } } else { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); STableNode* pTable = NULL; pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); - if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); + } + if (nodeType(pTable) != QUERY_NODE_REAL_TABLE) { + *pRewriteToColumn = true; + return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_TBNAME); } } return TSDB_CODE_SUCCESS; diff --git a/tests/develop-test/2-query/pseudo_column.py b/tests/develop-test/2-query/pseudo_column.py index 4a97be6612..1d94df4cff 100644 --- a/tests/develop-test/2-query/pseudo_column.py +++ b/tests/develop-test/2-query/pseudo_column.py @@ -55,7 +55,7 @@ class TDTestCase: tdSql.checkData(2, 0, 'ct2') tdSql.checkData(2, 0, 'ct2') - tdSql.query('select `st.tbname` from (select st.tbname from st) order by `st.tbname`') + tdSql.query('select tbname from (select st.tbname from st) order by tbname') tdSql.checkCols(1) tdSql.checkRows(4) tdSql.checkData(0, 0, 'ct1') @@ -63,6 +63,14 @@ class TDTestCase: tdSql.checkData(2, 0, 'ct2') tdSql.checkData(2, 0, 'ct2') + tdSql.query('select * from (select tbname, avg(f) from st partition by tbname) a partition by a.tbname order by a.tbname'); + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'ct1'); + tdSql.checkData(0, 1, 6.0); + tdSql.checkData(1, 0, 'ct2'); + tdSql.checkData(1, 1, 12.0); + tdSql.error('select tbname from (select * from st)') tdSql.error('select st.tbname from (select st.tbname from st)') tdSql.error('select `st.tbname` from (select st.tbname from st) order by tbname') From 34246ea0015abdb656352b311846d57bbe6d1ce1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 16 Oct 2023 14:22:40 +0800 Subject: [PATCH 31/47] fix: tag scan support limit and remove slimit optimization --- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/scanoperator.c | 29 ++-- source/libs/planner/src/planOptimizer.c | 37 ----- tests/develop-test/2-query/tag_scan.py | 206 ++++++++++++++++++++++++ tests/parallel_test/cases.task | 1 + 5 files changed, 221 insertions(+), 54 deletions(-) create mode 100644 tests/develop-test/2-query/tag_scan.py diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index d5d144ee65..69330a8aee 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -307,7 +307,6 @@ typedef struct STagScanInfo { SSDataBlock* pRes; SColMatchInfo matchInfo; int32_t curPos; - SLimitNode* pSlimit; SReadHandle readHandle; STableListInfo* pTableListInfo; uint64_t suid; @@ -318,6 +317,7 @@ typedef struct STagScanInfo { SArray* aUidTags; // SArray SArray* aFilterIdxs; // SArray SStorageAPI* pStorageAPI; + SLimitInfo limitInfo; } STagScanInfo; typedef enum EStreamScanMode { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8dbb8a979e..2388e3dac7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3060,7 +3060,12 @@ static SSDataBlock* doTagScanFromCtbIdx(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); } pRes->info.rows = count; - pOperator->resultInfo.totalRows += count; + + bool bLimitReached = applyLimitOffset(&pInfo->limitInfo, pRes, pTaskInfo); + if (bLimitReached) { + setOperatorCompleted(pOperator); + } + pOperator->resultInfo.totalRows += pRes->info.rows; return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -3094,28 +3099,20 @@ static SSDataBlock* doTagScanFromMetaEntry(SOperatorInfo* pOperator) { if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); } - // each table with tbname is a group, hence its own block, but only group when slimit exists for performance reason. - if (pInfo->pSlimit != NULL) { - if (pInfo->curPos < pInfo->pSlimit->offset) { - continue; - } - pInfo->pRes->info.id.groupId = calcGroupId(mr.me.name, strlen(mr.me.name)); - if (pInfo->curPos >= (pInfo->pSlimit->offset + pInfo->pSlimit->limit) - 1) { - setOperatorCompleted(pOperator); - } - break; - } } + pRes->info.rows = count; pAPI->metaReaderFn.clearReader(&mr); - + bool bLimitReached = applyLimitOffset(&pInfo->limitInfo, pRes, pTaskInfo); + if (bLimitReached) { + setOperatorCompleted(pOperator); + } // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { setTaskStatus(pTaskInfo, TASK_COMPLETED); } - pRes->info.rows = count; - pOperator->resultInfo.totalRows += count; + pOperator->resultInfo.totalRows += pRes->info.rows; return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -3169,8 +3166,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; - pInfo->pSlimit = (SLimitNode*)pPhyNode->node.pSlimit; //TODO: slimit now only indicate group + initLimitInfo(pPhyNode->node.pLimit, pPhyNode->node.pSlimit, &pInfo->limitInfo); setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); initResultSizeInfo(&pOperator->resultInfo, 4096); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 430e69f46f..8d4c042960 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2730,36 +2730,6 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) { return true; } -static SLogicNode* tagScanOptFindAncestorWithSlimit(SLogicNode* pTableScanNode) { - SLogicNode* pNode = pTableScanNode->pParent; - while (NULL != pNode) { - if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) || QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) || - QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) || QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pNode)) { - return NULL; - } - if (NULL != pNode->pSlimit) { - return pNode; - } - pNode = pNode->pParent; - } - return NULL; -} - -static void tagScanOptCloneAncestorSlimit(SLogicNode* pTableScanNode) { - if (NULL != pTableScanNode->pSlimit) { - return; - } - - SLogicNode* pNode = tagScanOptFindAncestorWithSlimit(pTableScanNode); - if (NULL != pNode) { - // TODO: only set the slimit now. push down slimit later - pTableScanNode->pSlimit = nodesCloneNode(pNode->pSlimit); - ((SLimitNode*)pTableScanNode->pSlimit)->limit += ((SLimitNode*)pTableScanNode->pSlimit)->offset; - ((SLimitNode*)pTableScanNode->pSlimit)->offset = 0; - } - return; -} - static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SScanLogicNode* pScanNode = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, tagScanOptShouldBeOptimized); if (NULL == pScanNode) { @@ -2795,13 +2765,6 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp pScanNode->node.pTargets = pScanTargets; } - int32_t code = replaceLogicNode(pLogicSubplan, pAgg, (SLogicNode*)pScanNode); - if (TSDB_CODE_SUCCESS == code) { - NODES_CLEAR_LIST(pAgg->pChildren); - } - nodesDestroyNode((SNode*)pAgg); - tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode); - pScanNode->onlyMetaCtbIdx = false; pCxt->optimized = true; diff --git a/tests/develop-test/2-query/tag_scan.py b/tests/develop-test/2-query/tag_scan.py new file mode 100644 index 0000000000..a853e497dd --- /dev/null +++ b/tests/develop-test/2-query/tag_scan.py @@ -0,0 +1,206 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-11204]Difference improvement that can ignore negative + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self._conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tagscan") + + + def runSingleVgroup(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tagscan2") + tdSql.execute("create database if not exists tagscan2 vgroups 1") + tdSql.execute('use tagscan2') + tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tb1 using stb1 tags(1,'1',1.0);") + + tdSql.execute("create table tb2 using stb1 tags(2,'2',2.0);") + + tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") + + tdSql.execute("create table tb4 using stb1 tags(4,'4',4.0);") + + tdSql.execute("create table tb5 using stb1 tags(5,'5',5.0);") + + tdSql.execute("create table tb6 using stb1 tags(5,'5',5.0);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb3 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb4 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb5 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb6 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.query('select tags t1,t2 from stb1 order by t1,t2;') + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, '1') + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, '2') + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, '3') + tdSql.checkData(3, 0, 4) + tdSql.checkData(3, 1, '4') + tdSql.checkData(4, 0, 5) + tdSql.checkData(4, 1, '5') + tdSql.checkData(5, 0, 5) + tdSql.checkData(5, 1, '5') + + tdSql.query('select * from (select tags t1,t2 from stb1 group by t1,t2 slimit 2,3) order by t1,t2;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tags tbname tn from stb1 group by tbname slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tbname tn from stb1 group by tbname slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tbname tn from stb1 group by tbname order by tbname limit 2,3) order by tn;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 'tb3') + + tdSql.query('select * from (select distinct tbname tn from stb1 limit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select distinct tbname tn, t1,t2 from stb1 limit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tags t1,t2 from stb1 order by t1, t2 limit 2,3) order by t1, t2;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 3) + tdSql.checkData(0, 1, '3') + tdSql.checkData(1, 0, 4) + tdSql.checkData(1, 1, '4') + tdSql.checkData(2, 0, 5) + tdSql.checkData(2, 1, '5') + + tdSql.query('select * from (select tbname tn, t1,t2 from stb1 partition by tbname slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tbname tn, t1,t2 from stb1 group by tbname, t1,t2 slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tags tbname tn, t1,t2 from stb1 group by tbname, t1,t2 slimit 2,3) order by tn;') + tdSql.checkRows(3) + + + tdSql.execute('drop database tagscan2') + def runMultiVgroups(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tagscan") + tdSql.execute("create database if not exists tagscan") + tdSql.execute('use tagscan') + tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tb1 using stb1 tags(1,'1',1.0);") + + tdSql.execute("create table tb2 using stb1 tags(2,'2',2.0);") + + tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") + + tdSql.execute("create table tb4 using stb1 tags(4,'4',4.0);") + + tdSql.execute("create table tb5 using stb1 tags(5,'5',5.0);") + + tdSql.execute("create table tb6 using stb1 tags(5,'5',5.0);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb3 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb4 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb5 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute('insert into tb6 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.query('select tags t1,t2 from stb1 order by t1,t2;') + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, '1') + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, '2') + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, '3') + tdSql.checkData(3, 0, 4) + tdSql.checkData(3, 1, '4') + tdSql.checkData(4, 0, 5) + tdSql.checkData(4, 1, '5') + tdSql.checkData(5, 0, 5) + tdSql.checkData(5, 1, '5') + + tdSql.query('select * from (select tags t1,t2 from stb1 group by t1,t2 slimit 2,3) order by t1,t2;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tags tbname tn from stb1 group by tbname slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tbname tn from stb1 group by tbname slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tbname tn from stb1 group by tbname order by tbname limit 2,3) order by tn;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 'tb3') + + tdSql.query('select * from (select distinct tbname tn from stb1 limit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select distinct tbname tn, t1,t2 from stb1 limit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tags t1,t2 from stb1 order by t1, t2 limit 2,3) order by t1, t2;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 3) + tdSql.checkData(0, 1, '3') + tdSql.checkData(1, 0, 4) + tdSql.checkData(1, 1, '4') + tdSql.checkData(2, 0, 5) + tdSql.checkData(2, 1, '5') + + tdSql.query('select * from (select tbname tn, t1,t2 from stb1 partition by tbname slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tbname tn, t1,t2 from stb1 group by tbname, t1,t2 slimit 2,3) order by tn;') + tdSql.checkRows(3) + + tdSql.query('select * from (select tags tbname tn, t1,t2 from stb1 group by tbname, t1,t2 slimit 2,3) order by tn;') + tdSql.checkRows(3) + + + tdSql.execute('drop database tagscan') + + def run(self): + self.runMultiVgroups() + self.runSingleVgroup() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e83586ca09..85cb8306cb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1271,6 +1271,7 @@ #develop test ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 2-query/ts-range.py +,,n,develop-test,python3 ./test.py -f 2-query/tag_scan.py ,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py From 7a23615b3fda7553c0ad17f6cee90f6b4d56920d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Oct 2023 17:53:48 +0800 Subject: [PATCH 32/47] fix:offset set to earliest clearly --- utils/test/c/replay_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/test/c/replay_test.c b/utils/test/c/replay_test.c index c105670733..92f3fe5102 100644 --- a/utils/test/c/replay_test.c +++ b/utils/test/c/replay_test.c @@ -27,6 +27,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "enable.replay", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); From 0a84d7a8b3780b3a243ccdb556f88ea19fb72d28 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 Oct 2023 19:20:48 +0800 Subject: [PATCH 33/47] fix stream snap deadlock --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 09fffa1f74..c6255be7cb 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -198,8 +198,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { taosWLockLatch(&pTq->pStreamMeta->lock); tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode)); - - taosWLockLatch(&pTq->pStreamMeta->lock); if (rollback) { tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); } else { From f5ea3649c5a4a78266c7a19bc980d040c691ee0f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 Oct 2023 19:26:08 +0800 Subject: [PATCH 34/47] fix stream snap deadlock --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index c6255be7cb..a406b8df34 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -211,8 +211,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { goto _err; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); - if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { code = -1; taosMemoryFree(pWriter); From c66ad0431dec1a669b9cc9f786f81b4b29442c2a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 16 Oct 2023 19:49:38 +0800 Subject: [PATCH 35/47] docs: release ver-3.2.0.0 --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 0e4785f643..fa6ec4df17 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.0.0.alpha") + SET(TD_VER_NUMBER "3.2.1.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 6f863d8c25..e43cd638f6 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.2.0.0 + + + ## 3.1.1.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 89bb8aaf8f..5ebf3af1eb 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.2.0.0 + + + ## 3.1.1.0 From b133418d88d43918c67f7363b83306c02a030fdf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 Oct 2023 20:03:44 +0800 Subject: [PATCH 36/47] fix stream snap deadlock --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index a406b8df34..e122cf19d3 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -206,10 +206,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { code = tdbPostCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); if (code) goto _err; } - if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - code = -1; - goto _err; - } if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { code = -1; From 4ca897246ce2ba883c92f4c417ccc232976b56e3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 12 Oct 2023 20:32:04 +0800 Subject: [PATCH 37/47] enh: proceed sync log buffer on failure of appending too --- source/libs/sync/src/syncMain.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index edecfcb2bc..eca499cf28 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2860,11 +2860,12 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){ } int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { + int32_t code = -1; if (pEntry->dataLen < sizeof(SMsgHead)) { sError("vgId:%d, cannot append an invalid client request with no msg head. type:%s, dataLen:%d", ths->vgId, TMSG_INFO(pEntry->originalRpcType), pEntry->dataLen); syncEntryDestroy(pEntry); - return -1; + goto _out; } // append to log buffer @@ -2873,9 +2874,11 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { ASSERT(terrno != 0); (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno, false); syncEntryDestroy(pEntry); - return -1; + goto _out; } - + + code = 0; +_out:; // proceed match index, with replicating on needed SyncIndex matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, NULL, "Append"); @@ -2886,7 +2889,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { // multi replica if (ths->replicaNum > 1) { - return 0; + return code; } // single replica @@ -2894,10 +2897,10 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit until commitIndex:%" PRId64 "", ths->vgId, ths->commitIndex); - return -1; + code = -1; } - return 0; + return code; } bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { From a16c9be49f41f8c4233171f4dda235e41e35ca6d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 15:34:16 +0800 Subject: [PATCH 38/47] feat:[TD-19291]Schemaless table name can be composed based on rule --- .../03-insert-data/30-influxdb-line.mdx | 5 +- .../03-insert-data/40-opentsdb-telnet.mdx | 5 +- .../03-insert-data/50-opentsdb-json.mdx | 5 +- docs/en/14-reference/12-config/index.md | 9 +++ .../13-schemaless/13-schemaless.md | 2 + ...-influxdb-line.mdx => 30-influxdb-line.md} | 5 +- .../03-insert-data/40-opentsdb-telnet.mdx | 5 +- .../03-insert-data/50-opentsdb-json.mdx | 5 +- docs/zh/14-reference/12-config/index.md | 11 +++- .../13-schemaless/13-schemaless.md | 7 ++- examples/rust/src/lib.rs | 11 ++++ include/common/tglobal.h | 1 + source/client/src/clientSml.c | 52 ++++++++++++---- source/common/src/tglobal.c | 9 ++- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/sml-TD19291.py | 60 +++++++++++++++++++ tests/system-test/2-query/sml.py | 2 +- utils/test/c/sml_test.c | 4 +- 18 files changed, 172 insertions(+), 27 deletions(-) rename docs/zh/07-develop/03-insert-data/{30-influxdb-line.mdx => 30-influxdb-line.md} (76%) create mode 100644 examples/rust/src/lib.rs create mode 100644 tests/system-test/2-query/sml-TD19291.py diff --git a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx index c85363c9db..bd430d5973 100644 --- a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx @@ -38,7 +38,10 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to NCHAR type automatically - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h) -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored +- The rule of table name + - The child table name is created automatically in a rule to guarantee its uniqueness. + - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4` + - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, smlDataFormat is discarded since 3.0.3.0) ::: diff --git a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx index 1147ce01b0..ed2659042f 100644 --- a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx @@ -33,7 +33,10 @@ For example: meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The rule of table name + - The child table name is created automatically in a rule to guarantee its uniqueness. + - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4` + - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. diff --git a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx index 3dd6c6b756..a40b5f264d 100644 --- a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -48,7 +48,10 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http :::note - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. -- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- The rule of table name + - The child table name is created automatically in a rule to guarantee its uniqueness. + - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4` + - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored ::: diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 011db81b92..ac2364a65d 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -652,6 +652,15 @@ The charset that takes effect is UTF-8. | Type | String | | Default Value | None | +### smlAutoChildTableNameDelimiter + +| Attribute | Description | +| ------------- | ------------------------------------------ | +| Applicable | Client only | +| Meaning | Delimiter between tags as table name| +| Type | String | +| Default Value | None | + ### smlTagName | Attribute | Description | diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index d4b6606ac5..11d3c0d0ab 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -93,6 +93,8 @@ Note that tag_key1, tag_key2 are not the original order of the tags entered by t The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has. ::: +If you do not want to use an automatically generated table name, there are two ways to specify sub table names, the first one has a higher priority. +You can configure smlAutoChildTableNameDelimiter in taos.cfg, for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. diff --git a/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/30-influxdb-line.md similarity index 76% rename from docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx rename to docs/zh/07-develop/03-insert-data/30-influxdb-line.md index eb17386fd0..ed84f05fef 100644 --- a/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/30-influxdb-line.md @@ -38,7 +38,10 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理 - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度 - 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false，从3.0.3.0开始，该配置废弃） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) -- 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 子表名生成规则 + - 默认产生的子表名是根据规则生成的唯一 ID 值。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: diff --git a/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx index 97cb0d4a11..195f3a31f4 100644 --- a/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx @@ -31,8 +31,11 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB ```txt meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` +- 子表名生成规则 + - 默认产生的子表名是根据规则生成的唯一 ID 值。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) -- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 参考 [OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。 ## 示例代码 diff --git a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx index a63579bb2c..ce8d5b0caa 100644 --- a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,10 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 :::note - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 -- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 +- 子表名生成规则 + - 默认产生的子表名是根据规则生成的唯一 ID 值。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。 + - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index c7323da271..27a9618107 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -648,7 +648,16 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅客户端适用 | | 含义 | schemaless 自定义的子表名的 key | | 类型 | 字符串 | -| 缺省值 | 无 | +| 缺省值 | 无 + +### smlAutoChildTableNameDelimiter + +| 属性 | 说明 | +| -------- | ------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | schemaless tag之间的连接符，连起来作为子表名 | +| 类型 | 字符串 | +| 缺省值 | 无 | ### smlTagName diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 969bc8c2ae..084e31d810 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -94,8 +94,11 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 :::tip 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 -:::tip -为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +:::tip +如果不想用自动生成的表名，有两种指定子表名的方式，第一种优先级更高： +通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定。 +举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 +通过在taos.cfg里配置 smlChildTableName 参数来指定。 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。 diff --git a/examples/rust/src/lib.rs b/examples/rust/src/lib.rs new file mode 100644 index 0000000000..4f8a5f1df8 --- /dev/null +++ b/examples/rust/src/lib.rs @@ -0,0 +1,11 @@ +#![allow(unused)] +#![allow(non_camel_case_types)] + +#[path = "bindings.rs"] +pub mod bindings; +#[path = "subscriber.rs"] +pub mod subscriber; +#[path = "tdengine.rs"] +pub mod tdengine; +#[path = "utils.rs"] +pub mod utils; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4644c38ec4..c4037ed2ea 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -179,6 +179,7 @@ extern char tsUdfdLdLibPath[]; // schemaless extern char tsSmlChildTableName[]; +extern char tsSmlAutoChildTableNameDelimiter[]; extern char tsSmlTagName[]; extern bool tsSmlDot2Underline; extern char tsSmlTsDefaultName[]; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 91c21fe344..17645fbec5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -193,20 +193,46 @@ cleanup: } static int32_t smlParseTableName(SArray *tags, char *childTableName) { - size_t childTableNameLen = strlen(tsSmlChildTableName); - if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; - - for (int i = 0; i < taosArrayGetSize(tags); i++) { - SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - // handle child table name - if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); - if(tsSmlDot2Underline){ - smlStrReplace(childTableName, strlen(childTableName)); + bool autoChildName = false; + size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); + if(delimiter > 0){ + size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + totalNameLen += tag->length; + } + if(totalNameLen < TSDB_TABLE_NAME_LEN){ + autoChildName = true; + } + } + if(autoChildName){ + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + strncat(childTableName, tag->value, tag->length); + if(i != taosArrayGetSize(tags) - 1){ + strcat(childTableName, tsSmlAutoChildTableNameDelimiter); + } + } + if(tsSmlDot2Underline){ + smlStrReplace(childTableName, strlen(childTableName)); + } + }else{ + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + if(tsSmlDot2Underline){ + smlStrReplace(childTableName, strlen(childTableName)); + } + taosArrayRemove(tags, i); + break; } - taosArrayRemove(tags, i); - break; } } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3d7b38161a..ed1776dc45 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -118,7 +118,8 @@ bool tsSmlDot2Underline = true; char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts"; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. - // If set to empty system will generate table name using MD5 hash. +char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; + // If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) // bool tsSmlDataFormat = false; // int32_t tsSmlBatchSize = 10000; @@ -439,7 +440,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlChildTableName", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1; @@ -931,6 +933,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } + tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN); @@ -1396,6 +1399,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); } else if (strcasecmp("smlChildTableName", name) == 0) { tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) { + tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN); } else if (strcasecmp("smlTagName", name) == 0) { tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); // } else if (strcasecmp("smlDataFormat", name) == 0) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e83586ca09..9e596fc7af 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -399,6 +399,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TD19291.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R diff --git a/tests/system-test/2-query/sml-TD19291.py b/tests/system-test/2-query/sml-TD19291.py new file mode 100644 index 0000000000..7d13cecb0f --- /dev/null +++ b/tests/system-test/2-query/sml-TD19291.py @@ -0,0 +1,60 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'smlAutoChildTableNameDelimiter': '-', 'fqdn': 'localhost', 'smlDot2Underline': 1}, 'fqdn': 'localhost'} + print("===================: ", updatecfgDict) + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def check(self): + conn = taos.connect() + dbname = "td19291" + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s precision 'us'" % dbname) + conn.select_db(dbname) + + lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t1=3i64,t2=4f64,dataModelName=ttt c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t1=3i64,t2=4f.64 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t1=ioiooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t2=q,t1=iooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + 'st,t2=a,t1=ooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4 c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', + ] + conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) + print("inserted") + + tdSql.query("select table_name from information_schema.ins_tables where type = 'CHILD_TABLE' order by table_name") + tdSql.checkRows(6) + tdSql.checkData(0, 0, "3i64-4f64-\"t3\"") + tdSql.checkData(1, 0, "3i64-4f64-ttt") + tdSql.checkData(2, 0, "3i64-4f_64") + tdSql.checkData(3, 0, "a-ooo3i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i643i64uuuuuuuuuuuuuuuuuuuuuuuuuu4") + tdSql.checkData(4, 0, "t_418c134a0f00c7f536886e132d5fbfff") + tdSql.checkData(5, 0, "t_cb0dbf4ee9c9052815c17fc6483b0139") + # tdSql.query(f"select * from td24559.stb order by _ts") + # tdSql.checkRows(4) + # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)") + # tdSql.checkData(3, 2, "GEOMETRYCOLLECTION (MULTIPOINT ((0.000000 0.000000), (1.000000 1.000000)), POINT (3.000000 4.000000), LINESTRING (2.000000 3.000000, 3.000000 4.000000))") + return + + def run(self): + tdSql.prepare() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 53ac85bd8e..0369f45723 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -9,7 +9,7 @@ from util.dnodes import * from util.common import * class TDTestCase: - updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'} + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'smlAutoChildTableNameDelimiter': '', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'} print("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9153706d23..64da4f83e3 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1678,8 +1678,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_td18789_Test(); ASSERT(!ret); - ret = sml_td24070_Test(); - ASSERT(!ret); +// ret = sml_td24070_Test(); +// ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); From 486227d34157452e166c748a4bdc9e46f89cb02f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 15:35:49 +0800 Subject: [PATCH 39/47] fix:rollback --- examples/rust/src/lib.rs | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 examples/rust/src/lib.rs diff --git a/examples/rust/src/lib.rs b/examples/rust/src/lib.rs deleted file mode 100644 index 4f8a5f1df8..0000000000 --- a/examples/rust/src/lib.rs +++ /dev/null @@ -1,11 +0,0 @@ -#![allow(unused)] -#![allow(non_camel_case_types)] - -#[path = "bindings.rs"] -pub mod bindings; -#[path = "subscriber.rs"] -pub mod subscriber; -#[path = "tdengine.rs"] -pub mod tdengine; -#[path = "utils.rs"] -pub mod utils; From d8c48dd3465ca5f02af4c4723ce1261a8d1c7990 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 15:36:59 +0800 Subject: [PATCH 40/47] fix:rollback --- .../03-insert-data/{30-influxdb-line.md => 30-influxdb-line.mdx} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/zh/07-develop/03-insert-data/{30-influxdb-line.md => 30-influxdb-line.mdx} (100%) diff --git a/docs/zh/07-develop/03-insert-data/30-influxdb-line.md b/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx similarity index 100% rename from docs/zh/07-develop/03-insert-data/30-influxdb-line.md rename to docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx From 5839e71f1fd7033e875aa9bf0661c0e790a690a1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Oct 2023 18:52:25 +0800 Subject: [PATCH 41/47] fix:test case error --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9e596fc7af..0d36de5553 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -399,7 +399,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TD19291.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml-TD19291.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R From ed544186b3f6e473b0382821d57b386570a7047f Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 18 Oct 2023 10:53:31 +0800 Subject: [PATCH 42/47] test: add 500ms sleep after create-stream for branch-3.0 --- tests/pytest/util/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index ea042829d6..9c45c09715 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -140,6 +140,7 @@ class TDCom: self.range_count = 5 self.default_interval = 5 self.stream_timeout = 12 + self.create_stream_sleep = 0.5 self.record_history_ts = str() self.precision = "ms" self.date_time = self.genTs(precision=self.precision)[0] @@ -881,6 +882,7 @@ class TDCom: stream_options += f" ignore update 0" if not use_except: tdSql.execute(f'create stream if not exists {stream_name} trigger at_once {stream_options} {fill_history} into {des_table} {subtable} as {source_sql} {fill};') + time.sleep(self.create_stream_sleep) return None else: return f'create stream if not exists {stream_name} {stream_options} {fill_history} into {des_table} {subtable} as {source_sql} {fill};' @@ -906,6 +908,7 @@ class TDCom: stream_options += f" ignore update 0" if not use_except: tdSql.execute(f'create stream if not exists {stream_name} {stream_options} {fill_history} into {des_table}{stb_field_name} {tags} {subtable} as {source_sql} {fill};') + time.sleep(self.create_stream_sleep) return None else: return f'create stream if not exists {stream_name} {stream_options} {fill_history} into {des_table}{stb_field_name} {tags} {subtable} as {source_sql} {fill};' @@ -1566,8 +1569,8 @@ class TDCom: res1 = tdSql.queryResult tdSql.query(sql2) res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + tdSql.sql = sql1 new_list = list() - if tag_value_list: res1 = self.float_handle(res1) res2 = self.float_handle(res2) @@ -1602,6 +1605,7 @@ class TDCom: tdSql.query(sql2) # res2 = tdSql.queryResult res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + tdSql.sql = sql1 if tag_value_list: res1 = self.float_handle(res1) @@ -1643,6 +1647,7 @@ class TDCom: tdSql.query(sql2) # res2 = tdSql.queryResult res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + tdSql.sql = sql1 if tag_value_list: res1 = self.float_handle(res1) From 5b69b584c3873af092d2737adff576a53874d315 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 18 Oct 2023 16:58:58 +0800 Subject: [PATCH 43/47] enh: add log for deploy mnode --- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 7840528db9..d25c6438e8 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -32,8 +32,13 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { if (!option.deploy) { *required = mmDeployRequired(pInput); + if (*required) { + dInfo("deploy mnode required. dnodeId:%d<=0, clusterId:%" PRId64 "<=0, localEp:%s==firstEp", + pInput->pData->dnodeId, pInput->pData->clusterId, tsLocalEp); + } } else { *required = true; + dInfo("deploy mnode required. option deploy:%d", option.deploy); } return 0; From 5852a0975e6c96bd5523511e80dffe3d7b1f56d5 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Mon, 16 Oct 2023 13:53:47 +0800 Subject: [PATCH 44/47] set fill history range --- source/libs/executor/inc/executorInt.h | 2 ++ source/libs/executor/src/filloperator.c | 13 +++++++++++-- tests/script/tsim/stream/ignoreCheckUpdate.sim | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 69330a8aee..288919d709 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -636,6 +636,7 @@ typedef struct SStreamFillSupporter { SSHashObj* pResMap; bool hasDelete; SStorageAPI* pAPI; + STimeWindow winRange; } SStreamFillSupporter; typedef struct SStreamFillOperatorInfo { @@ -770,6 +771,7 @@ SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo); bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType); bool compareVal(const char* v, const SStateKeys* pKey); +bool inWinRange(STimeWindow* range, STimeWindow* cur); int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, int32_t prevPosition, int32_t order); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 9fce058c4c..ecc2526dd9 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -950,7 +950,10 @@ static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if (inWinRange(&pFillSup->winRange, &st)) { + buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); } @@ -960,7 +963,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { uint64_t groupId = pBlock->info.id.groupId; SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; - if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) { + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); pFillInfo->pLinearInfo->winIndex++; @@ -1345,6 +1349,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { pInfo->pFillInfo->preRowKey = INT64_MIN; } + pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pInfo->pFillSup->winRange.ekey <= 0) { + pInfo->pFillSup->winRange.ekey = INT64_MAX; + } + switch (pBlock->info.type) { case STREAM_RETRIEVE: return pBlock; diff --git a/tests/script/tsim/stream/ignoreCheckUpdate.sim b/tests/script/tsim/stream/ignoreCheckUpdate.sim index 725251b081..108c845e4d 100644 --- a/tests/script/tsim/stream/ignoreCheckUpdate.sim +++ b/tests/script/tsim/stream/ignoreCheckUpdate.sim @@ -206,6 +206,8 @@ print create stream streams3 trigger at_once ignore update 1 into streamt3 as se sql create stream streams3 trigger at_once ignore update 1 into streamt3 as select _wstart c1, count(*) c2, max(b) c3 from st interval(10s); +sleep 2000 + sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,2); From 312da0ac03557239bb77f94d251881e3b153546a Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 19 Oct 2023 13:59:41 +0800 Subject: [PATCH 45/47] docs(driver-go): update tmq auto.offset.reset configuration --- docs/en/07-develop/07-tmq.mdx | 4 +- docs/en/14-reference/03-connector/05-go.mdx | 88 ++++++++++++--------- docs/examples/go/sub/main.go | 34 ++++---- docs/zh/07-develop/07-tmq.md | 4 +- docs/zh/08-connector/20-go.mdx | 88 ++++++++++++--------- 5 files changed, 122 insertions(+), 96 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 47d82c0833..7dd06c9ca2 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -422,7 +422,7 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go conf := &tmq.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -511,7 +511,7 @@ var cfg = new ConsumerConfig GourpId = "TDengine-TMQ-C#", TDConnectUser = "root", TDConnectPasswd = "taosdata", - AutoOffsetReset = "earliest" + AutoOffsetReset = "latest" MsgWithTableName = "true", TDConnectIp = "127.0.0.1", TDConnectPort = "6030" diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index b3d4857d75..a0be7a4a02 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -794,7 +794,7 @@ The TDengine Go Connector supports subscription functionality with the following ```go consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -870,6 +870,7 @@ package main import ( "fmt" "os" + "time" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" @@ -890,19 +891,16 @@ func main() { if err != nil { panic(err) } - if err != nil { - panic(err) - } consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "earliest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_client", - "enable.auto.commit": "false", - "msg.with.table.name": "true", + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", }) if err != nil { panic(err) @@ -915,10 +913,16 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } + go func() { + for { + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) + } + }() + for i := 0; i < 5; i++ { ev := consumer.Poll(500) if ev != nil { @@ -972,6 +976,7 @@ package main import ( "database/sql" "fmt" + "time" "github.com/taosdata/driver-go/v3/common" tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" @@ -995,7 +1000,7 @@ func main() { "td.connect.pass": "taosdata", "group.id": "example", "client.id": "example_consumer", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", }) if err != nil { panic(err) @@ -1004,29 +1009,34 @@ func main() { if err != nil { panic(err) } + + _, err = db.Exec("create table example_ws_tmq.t_all(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") + if err != nil { + panic(err) + } go func() { - _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," + - "c1 bool," + - "c2 tinyint," + - "c3 smallint," + - "c4 int," + - "c5 bigint," + - "c6 tinyint unsigned," + - "c7 smallint unsigned," + - "c8 int unsigned," + - "c9 bigint unsigned," + - "c10 float," + - "c11 double," + - "c12 binary(20)," + - "c13 nchar(20)" + - ")") - if err != nil { - panic(err) - } - _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") - if err != nil { - panic(err) + for { + _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) } + }() for i := 0; i < 5; i++ { ev := consumer.Poll(500) diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index ed335cfdea..41cb33e94d 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "time" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" @@ -27,15 +28,15 @@ func main() { panic(err) } consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "earliest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_client", - "enable.auto.commit": "false", - "msg.with.table.name": "true", + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", }) if err != nil { panic(err) @@ -48,12 +49,17 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } + go func() { + for { + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + time.Sleep(time.Microsecond * 100) + } + }() for i := 0; i < 5; i++ { - ev := consumer.Poll(0) + ev := consumer.Poll(500) if ev != nil { switch e := ev.(type) { case *tmqcommon.DataMessage: diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 3272edbbca..a542c844fe 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -421,7 +421,7 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go conf := &tmq.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -512,7 +512,7 @@ var cfg = new ConsumerConfig GourpId = "TDengine-TMQ-C#", TDConnectUser = "root", TDConnectPasswd = "taosdata", - AutoOffsetReset = "earliest" + AutoOffsetReset = "latest" MsgWithTableName = "true", TDConnectIp = "127.0.0.1", TDConnectPort = "6030" diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 90ef4d83ca..3994278eef 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -797,7 +797,7 @@ TDengine Go 连接器支持订阅功能，应用 API 如下： ```go consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ "group.id": "test", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", "td.connect.ip": "127.0.0.1", "td.connect.user": "root", "td.connect.pass": "taosdata", @@ -873,6 +873,7 @@ package main import ( "fmt" "os" + "time" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" @@ -893,19 +894,16 @@ func main() { if err != nil { panic(err) } - if err != nil { - panic(err) - } consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "earliest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_client", - "enable.auto.commit": "false", - "msg.with.table.name": "true", + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", }) if err != nil { panic(err) @@ -918,10 +916,16 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } + go func() { + for { + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) + } + }() + for i := 0; i < 5; i++ { ev := consumer.Poll(500) if ev != nil { @@ -975,6 +979,7 @@ package main import ( "database/sql" "fmt" + "time" "github.com/taosdata/driver-go/v3/common" tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" @@ -998,7 +1003,7 @@ func main() { "td.connect.pass": "taosdata", "group.id": "example", "client.id": "example_consumer", - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", }) if err != nil { panic(err) @@ -1007,29 +1012,34 @@ func main() { if err != nil { panic(err) } + + _, err = db.Exec("create table example_ws_tmq.t_all(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") + if err != nil { + panic(err) + } go func() { - _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," + - "c1 bool," + - "c2 tinyint," + - "c3 smallint," + - "c4 int," + - "c5 bigint," + - "c6 tinyint unsigned," + - "c7 smallint unsigned," + - "c8 int unsigned," + - "c9 bigint unsigned," + - "c10 float," + - "c11 double," + - "c12 binary(20)," + - "c13 nchar(20)" + - ")") - if err != nil { - panic(err) - } - _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") - if err != nil { - panic(err) + for { + _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") + if err != nil { + panic(err) + } + time.Sleep(time.Millisecond * 100) } + }() for i := 0; i < 5; i++ { ev := consumer.Poll(500) From 691bbff845052a99a960ad42d57ab2ec1acf8c8c Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Thu, 19 Oct 2023 14:06:39 +0800 Subject: [PATCH 46/47] docs(zh): update offset.reset default setting --- docs/zh/08-connector/26-rust.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 018552117e..37b1787707 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -447,7 +447,7 @@ consumer.unsubscribe().await; - `group.id`: 同一个消费者组，将以至少消费一次的方式进行消息负载均衡。 - `client.id`: 可选的订阅客户端识别项。 -- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认为从头订阅。注意，此选项在同一个 `group.id` 中仅生效一次。 +- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认值根据 TDengine 版本有所不同，详细参见 [数据订阅](https://docs.taosdata.com/develop/tmq/)。注意，此选项在同一个 `group.id` 中仅生效一次。 - `enable.auto.commit`: 当设置为 `true` 时，将启用自动标记模式，当对数据一致性不敏感时，可以启用此方式。 - `auto.commit.interval.ms`: 自动标记的时间间隔。 From 2297beea5c054e68bfc154ab3111163843a284ed Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Thu, 19 Oct 2023 14:06:48 +0800 Subject: [PATCH 47/47] docs(en): update offset.reset default setting --- docs/en/14-reference/03-connector/06-rust.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index a98683d43c..5a44b161cb 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -442,7 +442,7 @@ The following parameters can be configured for the TMQ DSN. Only `group.id` is m - `group.id`: Within a consumer group, load balancing is implemented by consuming messages on an at-least-once basis. - `client.id`: Subscriber client ID. -- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default is earliest. Note: This parameter is set per consumer group. +- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default value varies depending on the TDengine version. For details, see [Data Subscription](https://docs.tdengine.com/develop/tmq/). Note: This parameter is set per consumer group. - `enable.auto.commit`: Automatically commits. This can be enabled when data consistency is not essential. - `auto.commit.interval.ms`: Interval for automatic commits.