From 7ea23a33bc09a32eb293158c23dcaea81aa24513 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 9 May 2023 16:37:08 +0800 Subject: [PATCH 001/122] enh: optimize time line function validation --- include/libs/nodes/querynodes.h | 94 ++++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstCreater.c | 2 +- source/libs/parser/src/parTranslater.c | 39 ++++++---- source/libs/parser/src/parUtil.c | 11 +++ source/libs/parser/test/parSelectTest.cpp | 9 +++ 7 files changed, 98 insertions(+), 60 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 480912a8cf..54c9d88a5e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -241,6 +241,12 @@ typedef enum EFillMode { FILL_MODE_NEXT } EFillMode; +typedef enum ETimeLineMode { + TIME_LINE_NONE = 1, + TIME_LINE_MULTI, + TIME_LINE_GLOBAL, +} ETimeLineMode; + typedef struct SFillNode { ENodeType type; // QUERY_NODE_FILL EFillMode mode; @@ -263,50 +269,50 @@ typedef struct SCaseWhenNode { } SCaseWhenNode; typedef struct SSelectStmt { - ENodeType type; // QUERY_NODE_SELECT_STMT - bool isDistinct; - SNodeList* pProjectionList; - SNode* pFromTable; - SNode* pWhere; - SNodeList* pPartitionByList; - SNodeList* pTags; // for create stream - SNode* pSubtable; // for create stream - SNode* pWindow; - SNodeList* pGroupByList; // SGroupingSetNode - SNode* pHaving; - SNode* pRange; - SNode* pEvery; - SNode* pFill; - SNodeList* pOrderByList; // SOrderByExprNode - SLimitNode* pLimit; - SLimitNode* pSlimit; - STimeWindow timeRange; - char stmtName[TSDB_TABLE_NAME_LEN]; - uint8_t precision; - int32_t selectFuncNum; - int32_t returnRows; // EFuncReturnRows - bool isEmptyResult; - bool isTimeLineResult; - bool isSubquery; - bool hasAggFuncs; - bool hasRepeatScanFuncs; - bool hasIndefiniteRowsFunc; - bool hasMultiRowsFunc; - bool hasSelectFunc; - bool hasSelectValFunc; - bool hasOtherVectorFunc; - bool hasUniqueFunc; - bool hasTailFunc; - bool hasInterpFunc; - bool hasInterpPseudoColFunc; - bool hasLastRowFunc; - bool hasLastFunc; - bool hasTimeLineFunc; - bool hasUdaf; - bool hasStateKey; - bool onlyHasKeepOrderFunc; - bool groupSort; - bool tagScan; + ENodeType type; // QUERY_NODE_SELECT_STMT + bool isDistinct; + SNodeList* pProjectionList; + SNode* pFromTable; + SNode* pWhere; + SNodeList* pPartitionByList; + SNodeList* pTags; // for create stream + SNode* pSubtable; // for create stream + SNode* pWindow; + SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; + SNode* pRange; + SNode* pEvery; + SNode* pFill; + SNodeList* pOrderByList; // SOrderByExprNode + SLimitNode* pLimit; + SLimitNode* pSlimit; + STimeWindow timeRange; + char stmtName[TSDB_TABLE_NAME_LEN]; + uint8_t precision; + int32_t selectFuncNum; + int32_t returnRows; // EFuncReturnRows + ETimeLineMode timeLineResMode; + bool isEmptyResult; + bool isSubquery; + bool hasAggFuncs; + bool hasRepeatScanFuncs; + bool hasIndefiniteRowsFunc; + bool hasMultiRowsFunc; + bool hasSelectFunc; + bool hasSelectValFunc; + bool hasOtherVectorFunc; + bool hasUniqueFunc; + bool hasTailFunc; + bool hasInterpFunc; + bool hasInterpPseudoColFunc; + bool hasLastRowFunc; + bool hasLastFunc; + bool hasTimeLineFunc; + bool hasUdaf; + bool hasStateKey; + bool onlyHasKeepOrderFunc; + bool groupSort; + bool tagScan; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 0f4e7bde63..ed0f47f68d 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -667,7 +667,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { COPY_CHAR_ARRAY_FIELD(stmtName); COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(isEmptyResult); - COPY_SCALAR_FIELD(isTimeLineResult); + COPY_SCALAR_FIELD(timeLineResMode); COPY_SCALAR_FIELD(hasAggFuncs); COPY_SCALAR_FIELD(hasRepeatScanFuncs); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 850571eea1..4130ccc483 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -84,6 +84,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); +int32_t getTableTypeFromTableNode(SNode *pTable); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index c53721f865..9a9cd57aeb 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -828,7 +828,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->pProjectionList = pProjectionList; select->pFromTable = pTable; sprintf(select->stmtName, "%p", select); - select->isTimeLineResult = true; + select->timeLineResMode = TIME_LINE_GLOBAL; select->onlyHasKeepOrderFunc = true; select->timeRange = TSWINDOW_INITIALIZER; return (SNode*)select; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6545c33a27..7692629a9f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -746,18 +746,18 @@ static SNodeList* getProjectList(const SNode* pNode) { static bool isTimeLineQuery(SNode* pStmt) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - return ((SSelectStmt*)pStmt)->isTimeLineResult; + return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode); } else { return false; } } static bool isGlobalTimeLineQuery(SNode* pStmt) { - if (!isTimeLineQuery(pStmt)) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + return TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode; + } else { return false; } - SSelectStmt* pSelect = (SSelectStmt*)pStmt; - return NULL == pSelect->pPartitionByList || NULL != pSelect->pOrderByList; } static bool isPrimaryKeyImpl(SNode* pExpr) { @@ -1563,7 +1563,7 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && - !isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function requires valid time series input", pFunc->functionName); } @@ -2273,7 +2273,7 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } if (!pSelect->onlyHasKeepOrderFunc) { - pSelect->isTimeLineResult = false; + pSelect->timeLineResMode = TIME_LINE_NONE; } CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, .existCol = false}; nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); @@ -2562,9 +2562,9 @@ static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSele static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTable) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pLeft) && - !isTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || + !isGlobalTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || (QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pRight) && - !isTimeLineQuery(((STempTableNode*)pJoinTable->pRight)->pSubquery))) { + !isGlobalTimeLineQuery(((STempTableNode*)pJoinTable->pRight)->pSubquery))) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SUPPORT_JOIN, "Join requires valid time series input"); } @@ -2598,7 +2598,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { pCxt->stableQuery = true; } if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isSelectStmt(pCxt->pCurrStmt)) { - ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; + ((SSelectStmt*)pCxt->pCurrStmt)->timeLineResMode = TIME_LINE_NONE; } code = addNamespace(pCxt, pRealTable); } @@ -3071,7 +3071,7 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); } pCxt->currClause = SQL_CLAUSE_GROUP_BY; - pSelect->isTimeLineResult = false; + pSelect->timeLineResMode = TIME_LINE_NONE; return translateExprList(pCxt, pSelect->pGroupByList); } @@ -3471,7 +3471,18 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_PARTITION_BY; - int32_t code = translateExprList(pCxt, pSelect->pPartitionByList); + int32_t code = TSDB_CODE_SUCCESS; + + if (pSelect->pPartitionByList) { + int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable); + SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0); + if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && + 1 == pSelect->pPartitionByList->length && (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) { + pSelect->timeLineResMode = TIME_LINE_MULTI; + } + + code = translateExprList(pCxt, pSelect->pPartitionByList); + } if (TSDB_CODE_SUCCESS == code) { code = translateExprList(pCxt, pSelect->pTags); } @@ -3600,9 +3611,9 @@ static void resetResultTimeline(SSelectStmt* pSelect) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKey((STempTableNode*)pSelect->pFromTable, pOrder)) || (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) { - pSelect->isTimeLineResult = true; + pSelect->timeLineResMode = TIME_LINE_GLOBAL; } else { - pSelect->isTimeLineResult = false; + pSelect->timeLineResMode = TIME_LINE_NONE; } } @@ -6057,7 +6068,7 @@ static bool isEventWindowQuery(SSelectStmt* pSelect) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || - !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || + !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aab..a4062d8407 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -249,6 +249,17 @@ int32_t getNumOfTags(const STableMeta* pTableMeta) { return getTableInfo(pTableM STableComInfo getTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; } +int32_t getTableTypeFromTableNode(SNode *pTable) { + if (NULL == pTable) { + return -1; + } + if (QUERY_NODE_REAL_TABLE != nodeType(pTable)) { + return -1; + } + return ((SRealTableNode *)pTable)->pMeta->tableType; +} + + STableMeta* tableMetaDup(const STableMeta* pTableMeta) { int32_t numOfFields = TABLE_TOTAL_COL_NUM(pTableMeta); if (numOfFields > TSDB_MAX_COLUMNS || numOfFields < TSDB_MIN_COLUMNS) { diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 2d8ce55b72..f2ed7dfa3b 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -117,6 +117,15 @@ TEST_F(ParserSelectTest, timelineFunc) { run("SELECT LAST(*), FIRST(*) FROM t1 INTERVAL(10s)"); run("SELECT diff(c1) FROM t1"); + + run("select diff(ts) from (select _wstart as ts, count(*) from st1 partition by tbname interval(1d))", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + + run("select diff(ts) from (select _wstart as ts, count(*) from st1 partition by tbname interval(1d) order by ts)"); + + run("select t1.* from st1s1 t1, (select _wstart as ts, count(*) from st1s2 partition by tbname interval(1d)) WHERE t1.ts = t2.ts", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("select t1.* from st1s1 t1, (select _wstart as ts, count(*) from st1s2 partition by tbname interval(1d) order by ts) t2 WHERE t1.ts = t2.ts"); + } TEST_F(ParserSelectTest, selectFunc) { From eb7cd25e2ef6ad472a4fd725ba44bebf46f5b583 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 10 May 2023 14:34:41 +0800 Subject: [PATCH 002/122] enh: support nest partition by clause --- include/libs/nodes/nodes.h | 3 + source/libs/nodes/src/nodesMatchFuncs.c | 180 ++++++++++++++++++++++++ source/libs/parser/src/parTranslater.c | 15 ++ 3 files changed, 198 insertions(+) create mode 100755 source/libs/nodes/src/nodesMatchFuncs.c diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3556a8ac95..320a89030b 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -321,6 +321,8 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* p SNode* nodesListGetNode(SNodeList* pList, int32_t index); SListCell* nodesListGetCell(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); +bool nodesListMatch(const SNodeList* pList, const SNodeList* pSubList); + // Only clear the linked list structure, without releasing the elements inside void nodesClearList(SNodeList* pList); @@ -339,6 +341,7 @@ void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pCon void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNode* a, const SNode* b); +bool nodesMatchNode(const SNode* pSub, const SNode* pNode); SNode* nodesCloneNode(const SNode* pNode); SNodeList* nodesCloneList(const SNodeList* pList); diff --git a/source/libs/nodes/src/nodesMatchFuncs.c b/source/libs/nodes/src/nodesMatchFuncs.c new file mode 100755 index 0000000000..a91fdcdfac --- /dev/null +++ b/source/libs/nodes/src/nodesMatchFuncs.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "querynodes.h" + +#define MATCH_SCALAR_FIELD(fldname) \ + do { \ + if (p->fldname != pSub->fldname) return false; \ + } while (0) + +#define MATCH_STRING(a, b) (((a) != NULL && (b) != NULL) ? (strcmp((a), (b)) == 0) : (a) == (b)) + +#define MATCH_VARDATA(a, b) \ + (((a) != NULL && (b) != NULL) \ + ? (varDataLen((a)) == varDataLen((b)) && memcmp(varDataVal((a)), varDataVal((b)), varDataLen((a))) == 0) \ + : (a) == (b)) + +#define MATCH_STRING_FIELD(fldname) \ + do { \ + if (!MATCH_STRING(p->fldname, pSub->fldname)) return false; \ + } while (0) + +#define MATCH_VARDATA_FIELD(fldname) \ + do { \ + if (!MATCH_VARDATA(p->fldname, pSub->fldname)) return false; \ + } while (0) + +#define MATCH_OBJECT_FIELD(fldname, matchFunc) \ + do { \ + if (!matchFunc(p->fldname, pSub->fldname)) return false; \ + } while (0) + +#define MATCH_NODE_FIELD(fldname) \ + do { \ + if (!nodesMatchNode(pSub->fldname, p->fldname)) return false; \ + } while (0) + +#define MATCH_NODE_LIST_FIELD(fldname) \ + do { \ + if (!nodesListMatch(p->fldname, pSub->fldname)) return false; \ + } while (0) + + +bool nodesListMatchExists(const SNodeList* pList, const SNode* pTarget) { + if (NULL == pList || NULL == pTarget) { + return false; + } + SNode* node = NULL; + bool exists = false; + FOREACH(node, pList) { + if (nodesMatchNode(node, pTarget)) { + exists = true; + break; + } + } + + return exists; +} + +bool nodesListMatch(const SNodeList* pList, const SNodeList* pSubList) { + if (pList == pSubList) { + return true; + } + + if (NULL == pList || NULL == pSubList) { + return false; + } + + if (pList->length != pSubList->length) { + return false; + } + + SNode* node = NULL; + bool match = false; + FOREACH(node, pList) { + if (!nodesListMatchExists(pSubList, node)) { + break; + } + } + return match; +} + +static bool columnNodeMatch(const SColumnNode* pSub, const SColumnNode* p) { + MATCH_STRING_FIELD(dbName); + if (0 == strcmp(p->colName, pSub->node.aliasName)) { + return true; + } + return false; +} + +static bool valueNodeMatch(const SValueNode* pSub, const SValueNode* p) { + return nodesEqualNode((SNode*)pSub, (SNode*)p); +} + +static bool operatorNodeMatch(const SOperatorNode* pSub, const SOperatorNode* p) { + MATCH_SCALAR_FIELD(opType); + MATCH_NODE_FIELD(pLeft); + MATCH_NODE_FIELD(pRight); + return true; +} + +static bool logicConditionNodeMatch(const SLogicConditionNode* pSub, const SLogicConditionNode* p) { + MATCH_SCALAR_FIELD(condType); + MATCH_NODE_LIST_FIELD(pParameterList); + return true; +} + +static bool functionNodeMatch(const SFunctionNode* pSub, const SFunctionNode* p) { + MATCH_SCALAR_FIELD(funcId); + MATCH_STRING_FIELD(functionName); + MATCH_NODE_LIST_FIELD(pParameterList); + return true; +} + +static bool whenThenNodeMatch(const SWhenThenNode* pSub, const SWhenThenNode* p) { + MATCH_NODE_FIELD(pWhen); + MATCH_NODE_FIELD(pThen); + return true; +} + +static bool caseWhenNodeMatch(const SCaseWhenNode* pSub, const SCaseWhenNode* p) { + MATCH_NODE_FIELD(pCase); + MATCH_NODE_FIELD(pElse); + MATCH_NODE_LIST_FIELD(pWhenThenList); + return true; +} + +bool nodesMatchNode(const SNode* pSub, const SNode* p) { + if (pSub == p) { + return true; + } + + if (NULL == pSub || NULL == p) { + return false; + } + + if (nodeType(pSub) != nodeType(p)) { + return false; + } + + switch (nodeType(p)) { + case QUERY_NODE_COLUMN: + return columnNodeMatch((const SColumnNode*)pSub, (const SColumnNode*)p); + case QUERY_NODE_VALUE: + return valueNodeMatch((const SValueNode*)pSub, (const SValueNode*)p); + case QUERY_NODE_OPERATOR: + return operatorNodeMatch((const SOperatorNode*)pSub, (const SOperatorNode*)p); + case QUERY_NODE_LOGIC_CONDITION: + return logicConditionNodeMatch((const SLogicConditionNode*)pSub, (const SLogicConditionNode*)p); + case QUERY_NODE_FUNCTION: + return functionNodeMatch((const SFunctionNode*)pSub, (const SFunctionNode*)p); + case QUERY_NODE_WHEN_THEN: + return whenThenNodeMatch((const SWhenThenNode*)pSub, (const SWhenThenNode*)p); + case QUERY_NODE_CASE_WHEN: + return caseWhenNodeMatch((const SCaseWhenNode*)pSub, (const SCaseWhenNode*)p); + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + case QUERY_NODE_JOIN_TABLE: + case QUERY_NODE_GROUPING_SET: + case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: + return false; + default: + break; + } + + return false; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7692629a9f..9bf94308e1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -760,6 +760,21 @@ static bool isGlobalTimeLineQuery(SNode* pStmt) { } } +static bool isTimeLineAlignedQuery(SNode* pStmt) { + SSelectStmt *pSelect = (SSelectStmt *)pStmt; + SSelectStmt *pSub = (SSelectStmt *)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (isGlobalTimeLineQuery((SNode*)pSub)) { + return true; + } + if (!isTimeLineQuery((SNode*)pSub)) { + return false; + } + if (nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { + return true; + } + return false; +} + static bool isPrimaryKeyImpl(SNode* pExpr) { if (QUERY_NODE_COLUMN == nodeType(pExpr)) { return (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId); From 7d23b176e593cdbf198b77876248e803076ae5dc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 11 May 2023 09:09:32 +0800 Subject: [PATCH 003/122] meta/table: new api metaTrimTables for vnode split --- source/dnode/vnode/src/meta/metaTable.c | 44 ++++++++++++++++++++----- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3325f4055c..9562b1f462 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -837,22 +837,50 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi return 0; } +static void metaDropTables(SMeta *pMeta, SArray *tbUids) { + metaWLock(pMeta); + for (int i = 0; i < TARRAY_SIZE(tbUids); ++i) { + tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); + metaDropTableByUid(pMeta, uid, NULL); + metaDebug("batch drop table:%" PRId64, uid); + } + metaULock(pMeta); +} + +int metaTrimTables(SMeta *pMeta, int64_t version) { + int32_t code = 0; + + SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); + if (tbUids == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + // code = metaFilterTableByHash(pMeta, /*ttl, */ tbUids); + if (code != 0) { + goto end; + } + if (TARRAY_SIZE(tbUids) == 0) { + goto end; + } + + metaDropTables(pMeta, tbUids); + +end: + taosArrayDestroy(tbUids); + + return code; +} + int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { int ret = metaTtlSmaller(pMeta, ttl, tbUids); if (ret != 0) { return ret; } - if (taosArrayGetSize(tbUids) == 0) { + if (TARRAY_SIZE(tbUids) == 0) { return 0; } - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { - tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i); - metaDropTableByUid(pMeta, *uid, NULL); - metaDebug("ttl drop table:%" PRId64, *uid); - } - metaULock(pMeta); + metaDropTables(pMeta, tbUids); return 0; } From 30104f300ce0ad1b46ab4b86caa5983a15f84677 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 16 May 2023 17:32:54 +0800 Subject: [PATCH 004/122] enh: refactor func names of mndTransPerformFinishedStage and mndTrancCheckConflict --- source/dnode/mnode/impl/inc/mndTrans.h | 6 ++++-- source/dnode/mnode/impl/src/mndDb.c | 6 +++--- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- source/dnode/mnode/impl/src/mndIndex.c | 6 +++--- source/dnode/mnode/impl/src/mndMnode.c | 4 ++-- source/dnode/mnode/impl/src/mndSma.c | 4 ++-- source/dnode/mnode/impl/src/mndStb.c | 10 +++++----- source/dnode/mnode/impl/src/mndStream.c | 10 +++++----- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 14 +++++++------- 11 files changed, 35 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 03434573c4..0d70434622 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -78,8 +78,10 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbnam void mndTransSetSerial(STrans *pTrans); void mndTransSetParallel(STrans *pTrans); void mndTransSetOper(STrans *pTrans, EOperType oper); -int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans); - +int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans); +static int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { + return mndTransCheckConflict(pMnode, pTrans); +} int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e93b06fdea..15374a8ddd 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -572,7 +572,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mInfo("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mndTransSetDbName(pTrans, dbObj.name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetOper(pTrans, MND_OPER_CREATE_DB); if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; @@ -814,7 +814,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p int32_t code = -1; mndTransSetDbName(pTrans, pOld->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; @@ -1109,7 +1109,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { mInfo("trans:%d start to drop db:%s", pTrans->id, pDb->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cee3b3c61d..fba928fc81 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -632,7 +632,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); if (pTrans == NULL) goto _OVER; mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; @@ -889,7 +889,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; pRaw = mndDnodeActionEncode(pDnode); if (pRaw == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 83172acf64..2d2637b8ce 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -645,7 +645,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt // mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); @@ -721,7 +721,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; @@ -860,4 +860,4 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 19c3d59167..470bdffbb1 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -578,7 +578,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; SMnodeObj mnodeObj = {0}; mnodeObj.id = pDnode->id; @@ -732,7 +732,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 42ad9e24d5..996f6c6494 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -622,7 +622,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-sma"); if (pTrans == NULL) goto _OVER; mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create sma:%s stream:%s", pTrans->id, pCreate->name, streamObj.name); @@ -845,7 +845,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p mInfo("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8525724fad..c35e322220 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -874,7 +874,7 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) return -1; + if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1; if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; @@ -1968,7 +1968,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (needRsp) { void *pCont = NULL; @@ -1998,7 +1998,7 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (needRsp) { void *pCont = NULL; @@ -2242,7 +2242,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p mInfo("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; @@ -3298,7 +3298,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0713150b48..53577d2b91 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -733,7 +733,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndTransDrop(pTrans); goto _OVER; } @@ -888,7 +888,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "stream-checkpoint"); if (pTrans == NULL) return -1; mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndReleaseStream(pMnode, pStream); mndTransDrop(pTrans); return -1; @@ -999,7 +999,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return -1; @@ -1367,7 +1367,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return -1; @@ -1475,7 +1475,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77..e2126ad103 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -476,7 +476,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu } mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndTransDrop(pTrans); return -1; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f6da370916..aaac0abb50 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -740,7 +740,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } mndTransSetDbName(pTrans, pTopic->db, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index cfb5bef9d0..16f58be85f 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -52,7 +52,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); +static bool mndTransPerformFinishedStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); @@ -872,7 +872,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { return conflict; } -int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { +int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) { if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { if (strlen(pTrans->dbname) == 0 && strlen(pTrans->stbname) == 0) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; @@ -891,7 +891,7 @@ int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { return -1; } @@ -1528,7 +1528,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerfromPreFinishedStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { if (mndCannotExecuteTransAction(pMnode)) return false; bool continueExec = true; @@ -1547,7 +1547,7 @@ static bool mndTransPerfromPreFinishedStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformFinishedStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = false; SSdbRaw *pRaw = mndTransActionEncode(pTrans); @@ -1605,14 +1605,14 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { break; case TRN_STAGE_PRE_FINISH: if (isLeader) { - continueExec = mndTransPerfromPreFinishedStage(pMnode, pTrans); + continueExec = mndTransPerformPreFinishStage(pMnode, pTrans); } else { mInfo("trans:%d, can not pre-finish since not leader", pTrans->id); continueExec = false; } break; case TRN_STAGE_FINISHED: - continueExec = mndTransPerfromFinishedStage(pMnode, pTrans); + continueExec = mndTransPerformFinishedStage(pMnode, pTrans); break; default: continueExec = false; From f8841046bcf21c8602f74c7b2509ffa7724cfd69 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 16 May 2023 19:00:08 +0800 Subject: [PATCH 005/122] enh: refactor func name of mndTransPerformFinishStage --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b14f7a9023..4ce7bb94eb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -108,7 +108,7 @@ typedef enum { TRN_STAGE_UNDO_ACTION = 3, TRN_STAGE_COMMIT = 4, TRN_STAGE_COMMIT_ACTION = 5, - TRN_STAGE_FINISHED = 6, + TRN_STAGE_FINISH = 6, TRN_STAGE_PRE_FINISH = 7 } ETrnStage; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 16f58be85f..d23662072d 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -52,7 +52,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformFinishedStage(SMnode *pMnode, STrans *pTrans); +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); @@ -458,7 +458,7 @@ static const char *mndTransStr(ETrnStage stage) { return "commit"; case TRN_STAGE_COMMIT_ACTION: return "commitAction"; - case TRN_STAGE_FINISHED: + case TRN_STAGE_FINISH: return "finished"; case TRN_STAGE_PRE_FINISH: return "pre-finish"; @@ -607,7 +607,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { } if (pOld->stage == TRN_STAGE_PRE_FINISH) { - pOld->stage = TRN_STAGE_FINISHED; + pOld->stage = TRN_STAGE_FINISH; mTrace("trans:%d, stage from pre-finish to finished since perform update action", pNew->id); } @@ -961,7 +961,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { bool sendRsp = false; int32_t code = pTrans->code; - if (pTrans->stage == TRN_STAGE_FINISHED) { + if (pTrans->stage == TRN_STAGE_FINISH) { sendRsp = true; } @@ -1476,7 +1476,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->code = 0; - pTrans->stage = TRN_STAGE_FINISHED; // TRN_STAGE_PRE_FINISH is not necessary + pTrans->stage = TRN_STAGE_FINISH; // TRN_STAGE_PRE_FINISH is not necessary mInfo("trans:%d, stage from commitAction to finished", pTrans->id); continueExec = true; } else { @@ -1535,7 +1535,7 @@ static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransPreFinish(pMnode, pTrans); if (code == 0) { - pTrans->stage = TRN_STAGE_FINISHED; + pTrans->stage = TRN_STAGE_FINISH; mInfo("trans:%d, stage from pre-finish to finish", pTrans->id); continueExec = true; } else { @@ -1547,7 +1547,7 @@ static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformFinishedStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = false; SSdbRaw *pRaw = mndTransActionEncode(pTrans); @@ -1611,8 +1611,8 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = false; } break; - case TRN_STAGE_FINISHED: - continueExec = mndTransPerformFinishedStage(pMnode, pTrans); + case TRN_STAGE_FINISH: + continueExec = mndTransPerformFinishStage(pMnode, pTrans); break; default: continueExec = false; From e70c433a5911130caf9274ab3c3d98d7693775b6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 26 May 2023 08:44:23 +0800 Subject: [PATCH 006/122] enh: support time line query from union/union all subquery --- include/libs/nodes/querynodes.h | 1 + source/libs/executor/src/sortoperator.c | 1 + source/libs/nodes/src/nodesMatchFuncs.c | 4 +-- source/libs/parser/src/parTranslater.c | 36 +++++++++++++++++++---- source/libs/parser/test/parSelectTest.cpp | 4 +++ 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 4b46808c5a..12890571f9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -327,6 +327,7 @@ typedef struct SSetOperator { SNode* pLimit; char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; + ETimeLineMode timeLineResMode; } SSetOperator; typedef enum ESqlClause { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 718bb596c5..f6827e4a5f 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -153,6 +153,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } + pDataBlock->info.dataLoad = 1; pDataBlock->info.rows = p->info.rows; } diff --git a/source/libs/nodes/src/nodesMatchFuncs.c b/source/libs/nodes/src/nodesMatchFuncs.c index a91fdcdfac..401c7aad28 100755 --- a/source/libs/nodes/src/nodesMatchFuncs.c +++ b/source/libs/nodes/src/nodesMatchFuncs.c @@ -83,9 +83,10 @@ bool nodesListMatch(const SNodeList* pList, const SNodeList* pSubList) { } SNode* node = NULL; - bool match = false; + bool match = true; FOREACH(node, pList) { if (!nodesListMatchExists(pSubList, node)) { + match = false; break; } } @@ -93,7 +94,6 @@ bool nodesListMatch(const SNodeList* pList, const SNodeList* pSubList) { } static bool columnNodeMatch(const SColumnNode* pSub, const SColumnNode* p) { - MATCH_STRING_FIELD(dbName); if (0 == strcmp(p->colName, pSub->node.aliasName)) { return true; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ce28848b9c..42717765f6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -752,6 +752,8 @@ static SNodeList* getProjectList(const SNode* pNode) { static bool isTimeLineQuery(SNode* pStmt) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode); + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { + return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; } else { return false; } @@ -760,6 +762,8 @@ static bool isTimeLineQuery(SNode* pStmt) { static bool isGlobalTimeLineQuery(SNode* pStmt) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode; + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { + return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; } else { return false; } @@ -767,13 +771,16 @@ static bool isGlobalTimeLineQuery(SNode* pStmt) { static bool isTimeLineAlignedQuery(SNode* pStmt) { SSelectStmt *pSelect = (SSelectStmt *)pStmt; - SSelectStmt *pSub = (SSelectStmt *)((STempTableNode*)pSelect->pFromTable)->pSubquery; - if (isGlobalTimeLineQuery((SNode*)pSub)) { + if (isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return true; } - if (!isTimeLineQuery((SNode*)pSub)) { + if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return false; } + if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return false; + } + SSelectStmt *pSub = (SSelectStmt *)((STempTableNode*)pSelect->pFromTable)->pSubquery; if (nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { return true; } @@ -1582,7 +1589,7 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && - !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && !isTimeLineAlignedQuery(pCxt->pCurrStmt)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function requires valid time series input", pFunc->functionName); } @@ -3828,8 +3835,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS pLeftExpr = pLeftFuncExpr; } snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); - if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, - createSetOperProject(pSetOperator->stmtName, pLeft))) { + SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); + if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) + && ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID + && ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + ((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { return TSDB_CODE_OUT_OF_MEMORY; } } @@ -3841,6 +3853,10 @@ static uint8_t calcSetOperatorPrecision(SSetOperator* pSetOperator) { } static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pSetOperator) { + if (NULL == pSetOperator->pOrderByList || pSetOperator->pOrderByList->length <= 0) { + return TSDB_CODE_SUCCESS; + } + bool other; int32_t code = translateOrderByPosition(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList, &other); if (TSDB_CODE_SUCCESS == code) { @@ -3853,6 +3869,14 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS if (TSDB_CODE_SUCCESS == code) { code = replaceOrderByAlias(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList); } + if (TSDB_CODE_SUCCESS == code) { + SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSetOperator->timeLineResMode = TIME_LINE_GLOBAL; + } else { + pSetOperator->timeLineResMode = TIME_LINE_NONE; + } + } return code; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index f2ed7dfa3b..d1d5b00037 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -334,6 +334,10 @@ TEST_F(ParserSelectTest, subquery) { run("SELECT SUM(a) FROM (SELECT MAX(c1) a, _wstart FROM st1s1 PARTITION BY TBNAME INTERVAL(1m) ORDER BY _WSTART) " "INTERVAL(1n)"); + run("SELECT diff(a) FROM (SELECT _wstart, tag1, tag2, MAX(c1) a FROM st1 PARTITION BY tag1 INTERVAL(1m)) PARTITION BY tag1"); + + run("SELECT diff(a) FROM (SELECT _wstart, tag1, tag2, MAX(c1) a FROM st1 PARTITION BY tag1 INTERVAL(1m)) PARTITION BY tag2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + run("SELECT _C0 FROM (SELECT _ROWTS, ts FROM st1s1)"); run("SELECT ts FROM (SELECT t1.ts FROM st1s1 t1)"); From ee4709bb4b8c30ba2fe9f997496628b0d169d8c6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 26 May 2023 16:01:42 +0800 Subject: [PATCH 007/122] chore: update docs about support row/columns width up to 64KB --- docs/en/12-taos-sql/01-data-type.md | 2 +- docs/en/12-taos-sql/03-table.md | 2 +- docs/en/12-taos-sql/19-limit.md | 2 +- docs/en/14-reference/13-schemaless/13-schemaless.md | 2 +- docs/zh/12-taos-sql/01-data-type.md | 4 ++-- docs/zh/12-taos-sql/03-table.md | 2 +- docs/zh/12-taos-sql/19-limit.md | 2 +- docs/zh/14-reference/13-schemaless/13-schemaless.md | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index 70bea97dba..13007d5bb1 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -45,7 +45,7 @@ In TDengine, the data types below can be used when specifying a column or tag. :::note - Only ASCII visible characters are suggested to be used in a column or tag of BINARY type. Multi-byte characters must be stored in NCHAR type. -- The length of BINARY can be up to 16,374 bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` +- The length of BINARY can be up to 16,374(data column is 65,517 and tag column is 16,382 since version 3.0.5.0) bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` - Numeric values in SQL statements will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used, so attention must be paid to avoid overflow. For example, 9999999999999999999 will be considered as overflow because it exceeds the upper limit of long integer, but 9999999999999999999.0 will be considered as a legal float number. ::: diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index f61d1f5147..7f39fb5867 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -45,7 +45,7 @@ table_option: { 1. The first column of a table MUST be of type TIMESTAMP. It is automatically set as the primary key. 2. The maximum length of the table name is 192 bytes. -3. The maximum length of each row is 48k bytes, please note that the extra 2 bytes used by each BINARY/NCHAR column are also counted. +3. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR column are also counted. 4. The name of the subtable can only consist of characters from the English alphabet, digits and underscore. Table names can't start with a digit. Table names are case insensitive. 5. The maximum length in bytes must be specified when using BINARY or NCHAR types. 6. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. diff --git a/docs/en/12-taos-sql/19-limit.md b/docs/en/12-taos-sql/19-limit.md index 654fae7560..22ad2055e4 100644 --- a/docs/en/12-taos-sql/19-limit.md +++ b/docs/en/12-taos-sql/19-limit.md @@ -26,7 +26,7 @@ The following characters cannot occur in a password: single quotation marks ('), - Maximum length of database name is 64 bytes - Maximum length of table name is 192 bytes, excluding the database name prefix and the separator. -- Maximum length of each data row is 48K bytes. Note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. +- Maximum length of each data row is 48K(64K since version 3.0.5.0) bytes. Note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. - The maximum length of a column name is 64 bytes. - Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp. - The maximum length of a tag name is 64 bytes diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index aad0e63a42..3ae9098a73 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -90,7 +90,7 @@ You can configure smlChildTableName in taos.cfg to specify table names, for exam Note: TDengine 3.0.3.0 and later automatically detect whether order is consistent. This parameter is no longer used. :::tip -All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. +All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB(64 KB since version 3.0.5.0) and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. ::: ## Time resolution recognition diff --git a/docs/zh/12-taos-sql/01-data-type.md b/docs/zh/12-taos-sql/01-data-type.md index f014573ca6..4a4c1d6ec6 100644 --- a/docs/zh/12-taos-sql/01-data-type.md +++ b/docs/zh/12-taos-sql/01-data-type.md @@ -45,9 +45,9 @@ CREATE DATABASE db_name PRECISION 'ns'; :::note -- 表的每行长度不能超过 48KB（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 +- 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 - 虽然 BINARY 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 BINARY 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符，虽然有时也能正常读写，但并不带有字符集信息，很容易出现数据乱码甚至数据损坏等情况。 -- BINARY 类型理论上最长可以有 16,374 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 +- BINARY 类型理论上最长可以有 16,374（从 3.0.5.0 版本开始，数据列为 65,517，标签列为 16,382） 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 - SQL 语句中的数值类型将依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999 会认为超过长整型的上边界而溢出，而 9999999999999999999.0 会被认为是有效的浮点数。 ::: diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 5687c7e740..2e66ac4002 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -43,7 +43,7 @@ table_option: { 1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键； 2. 表名最大长度为 192； -3. 表的每行长度不能超过 48KB;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） 4. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 5. 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； 6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 7b6692f1b7..e5a492580e 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -26,7 +26,7 @@ description: 合法字符集和命名中的限制规则 - 数据库名最大长度为 64 字节 - 表名最大长度为 192 字节，不包括数据库名前缀和分隔符 -- 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +- 每行数据最大长度 48KB（从 3.0.5.0 版本开始为 64KB） （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） - 列名最大长度为 64 字节 - 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 - 标签名最大长度为 64 字节 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index e5f232c1fc..6c2007938b 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -87,7 +87,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 :::tip 无模式所有的处理逻辑，仍会遵循 TDengine 对数据结构的底层限制，例如每行数据的总长度不能超过 -48KB，标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit) +48KB（从 3.0.5.0 版本开始为 64KB），标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit) ::: From 873ad8839bed1e23dffbf375bd4fa9dfc4a7a43e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 22 May 2023 10:37:55 +0800 Subject: [PATCH 008/122] enh: add prepareActions in STrans --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndTrans.h | 4 +- source/dnode/mnode/impl/src/mndSync.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 375 ++++++++++-------------- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- 5 files changed, 170 insertions(+), 216 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4ce7bb94eb..cd04b989ae 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -157,6 +157,7 @@ typedef struct { void* rpcRsp; int32_t rpcRspLen; int32_t redoActionPos; + SArray* prepareActions; SArray* redoActions; SArray* undoActions; SArray* commitActions; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 0d70434622..9fa59b4ac6 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -70,6 +70,7 @@ int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendNullLog(STrans *pTrans); +int32_t mndTransAppendPrepareAction(STrans *pTrans, STransAction *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); @@ -86,7 +87,8 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); -void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader); +void mndTransExecute(SMnode *pMnode, STrans *pTrans); +void mndTransRefresh(SMnode *pMnode, STrans *pTrans); int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0a6df02f5f..02925f9d8c 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -116,7 +116,7 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta if (pTrans != NULL) { mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", transId, pTrans->createdTime, pMgmt->transId); - mndTransExecute(pMnode, pTrans, false); + mndTransRefresh(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); } else { mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d23662072d..69a6053ba6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -23,15 +23,16 @@ #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 +#define TRANS_VER1_NUMBER 1 +#define TRANS_VER2_NUMBER 2 #define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 48 -static SSdbRaw *mndTransActionEncode(STrans *pTrans); -static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); +static SSdbRaw *mndTransEncode(STrans *pTrans); +static SSdbRow *mndTransDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); +static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); @@ -67,11 +68,11 @@ int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_TRANS, .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .encodeFp = (SdbEncodeFp)mndTransEncode, + .decodeFp = (SdbDecodeFp)mndTransDecode, .insertFp = (SdbInsertFp)mndTransActionInsert, .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete, + .deleteFp = (SdbDeleteFp)mndTransDelete, }; mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer); @@ -103,15 +104,55 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { return rawDataLen; } -static SSdbRaw *mndTransActionEncode(STrans *pTrans) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + +static int32_t mndTransEncodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pActions, int32_t actionsNum) { + int32_t dataPos = *offset; + int8_t unused = 0; + int32_t ret = -1; + + for (int32_t i = 0; i < actionsNum; ++i) { + STransAction *pAction = taosArrayGet(pActions, i); + SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) + if (pAction->actionType == TRANS_ACTION_RAW) { + int32_t len = sdbGetRawTotalSize(pAction->pRaw); + SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) + } else if (pAction->actionType == TRANS_ACTION_MSG) { + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) + SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing + } + } + ret = 0; + +_OVER: + *offset = dataPos; + return ret; +} + +static SSdbRaw *mndTransEncode(STrans *pTrans) { + terrno = TSDB_CODE_INVALID_MSG; + int8_t sver = taosArrayGetSize(pTrans->prepareActions) ? TRANS_VER2_NUMBER : TRANS_VER1_NUMBER; int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen; + rawDataLen += mndTransGetActionsSize(pTrans->prepareActions); rawDataLen += mndTransGetActionsSize(pTrans->redoActions); rawDataLen += mndTransGetActionsSize(pTrans->undoActions); rawDataLen += mndTransGetActionsSize(pTrans->commitActions); - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, sver, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -131,91 +172,22 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, pTrans->stbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pTrans->redoActionPos, _OVER) + int32_t prepareActionNum = taosArrayGetSize(pTrans->prepareActions); int32_t redoActionNum = taosArrayGetSize(pTrans->redoActions); int32_t undoActionNum = taosArrayGetSize(pTrans->undoActions); int32_t commitActionNum = taosArrayGetSize(pTrans->commitActions); + + if (sver > TRANS_VER1_NUMBER) { + SDB_SET_INT32(pRaw, dataPos, prepareActionNum, _OVER) + } SDB_SET_INT32(pRaw, dataPos, redoActionNum, _OVER) SDB_SET_INT32(pRaw, dataPos, undoActionNum, _OVER) SDB_SET_INT32(pRaw, dataPos, commitActionNum, _OVER) - int8_t unused = 0; - for (int32_t i = 0; i < redoActionNum; ++i) { - STransAction *pAction = taosArrayGet(pTrans->redoActions, i); - SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) - if (pAction->actionType == TRANS_ACTION_RAW) { - int32_t len = sdbGetRawTotalSize(pAction->pRaw); - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else if (pAction->actionType == TRANS_ACTION_MSG) { - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) - } else { - // nothing - } - } - - for (int32_t i = 0; i < undoActionNum; ++i) { - STransAction *pAction = taosArrayGet(pTrans->undoActions, i); - SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) - if (pAction->actionType == TRANS_ACTION_RAW) { - int32_t len = sdbGetRawTotalSize(pAction->pRaw); - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else if (pAction->actionType == TRANS_ACTION_MSG) { - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) - } else { - // nothing - } - } - - for (int32_t i = 0; i < commitActionNum; ++i) { - STransAction *pAction = taosArrayGet(pTrans->commitActions, i); - SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) - if (pAction->actionType == TRANS_ACTION_RAW) { - int32_t len = sdbGetRawTotalSize(pAction->pRaw); - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else if (pAction->actionType == TRANS_ACTION_MSG) { - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) - } else { - // nothing - } - } + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->prepareActions, prepareActionNum) < 0) goto _OVER; + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->redoActions, redoActionNum) < 0) goto _OVER; + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->undoActions, undoActionNum) < 0) goto _OVER; + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->commitActions, commitActionNum) < 0) goto _OVER; SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, _OVER) SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, _OVER) @@ -242,14 +214,67 @@ _OVER: return pRaw; } -static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { - terrno = TSDB_CODE_OUT_OF_MEMORY; +static int32_t mndTransDecodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pActions, int32_t actionNum) { + STransAction action = {0}; + int32_t dataPos = *offset; + int8_t unused = 0; + int8_t stage = 0; + int8_t actionType = 0; + int32_t dataLen = 0; + int32_t ret = -1; + + for (int32_t i = 0; i < actionNum; ++i) { + memset(&action, 0, sizeof(action)); + SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; + SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) + action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) + if (action.actionType == TRANS_ACTION_RAW) { + SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) + action.pRaw = taosMemoryMalloc(dataLen); + if (action.pRaw == NULL) goto _OVER; + mTrace("raw:%p, is created", action.pRaw); + SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); + if (taosArrayPush(pActions, &action) == NULL) goto _OVER; + action.pRaw = NULL; + } else if (action.actionType == TRANS_ACTION_MSG) { + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + tmsgUpdateDnodeEpSet(&action.epSet); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) + SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) + action.pCont = taosMemoryMalloc(action.contLen); + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pActions, &action) == NULL) goto _OVER; + action.pCont = NULL; + } else { + if (taosArrayPush(pActions, &action) == NULL) goto _OVER; + } + } + ret = 0; + +_OVER: + *offset = dataPos; + return ret; +} + +static SSdbRow *mndTransDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_INVALID_MSG; SSdbRow *pRow = NULL; STrans *pTrans = NULL; char *pData = NULL; int32_t dataLen = 0; int8_t sver = 0; + int32_t prepareActionNum = 0; int32_t redoActionNum = 0; int32_t undoActionNum = 0; int32_t commitActionNum = 0; @@ -258,7 +283,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TRANS_VER_NUMBER) { + if (sver != TRANS_VER1_NUMBER && sver != TRANS_VER2_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -294,127 +319,28 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->stbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pTrans->redoActionPos, _OVER) + + if (sver > TRANS_VER1_NUMBER) { + SDB_GET_INT32(pRaw, dataPos, &prepareActionNum, _OVER) + } SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) SDB_GET_INT32(pRaw, dataPos, &commitActionNum, _OVER) + pTrans->prepareActions = taosArrayInit(prepareActionNum, sizeof(STransAction)); pTrans->redoActions = taosArrayInit(redoActionNum, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(undoActionNum, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(commitActionNum, sizeof(STransAction)); + if (pTrans->prepareActions == NULL) goto _OVER; if (pTrans->redoActions == NULL) goto _OVER; if (pTrans->undoActions == NULL) goto _OVER; if (pTrans->commitActions == NULL) goto _OVER; - int8_t unused = 0; - for (int32_t i = 0; i < redoActionNum; ++i) { - memset(&action, 0, sizeof(action)); - SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) - action.actionType = actionType; - SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) - action.stage = stage; - SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) - if (action.actionType == TRANS_ACTION_RAW) { - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) - action.pRaw = taosMemoryMalloc(dataLen); - if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); - SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - action.pRaw = NULL; - } else if (action.actionType == TRANS_ACTION_MSG) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); - tmsgUpdateDnodeEpSet(&action.epSet); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) - action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto _OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - action.pCont = NULL; - } else { - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - } - } - - for (int32_t i = 0; i < undoActionNum; ++i) { - memset(&action, 0, sizeof(action)); - SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) - action.actionType = actionType; - SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) - action.stage = stage; - SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) - if (action.actionType == TRANS_ACTION_RAW) { - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) - action.pRaw = taosMemoryMalloc(dataLen); - if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); - SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; - action.pRaw = NULL; - } else if (action.actionType == TRANS_ACTION_MSG) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) - action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto _OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; - action.pCont = NULL; - } else { - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; - } - } - - for (int32_t i = 0; i < commitActionNum; ++i) { - memset(&action, 0, sizeof(action)); - SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) - action.actionType = actionType; - SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) - action.stage = stage; - SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) - if (action.actionType) { - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) - action.pRaw = taosMemoryMalloc(dataLen); - if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); - SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); - if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; - action.pRaw = NULL; - } else if (action.actionType == TRANS_ACTION_MSG) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) - action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto _OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); - if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; - action.pCont = NULL; - } else { - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - } - } + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->prepareActions, prepareActionNum) < 0) goto _OVER; + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->redoActions, redoActionNum) < 0) goto _OVER; + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->undoActions, undoActionNum) < 0) goto _OVER; + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->commitActions, commitActionNum) < 0) goto _OVER; SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, _OVER) SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, _OVER) @@ -520,6 +446,10 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { } static void mndTransDropData(STrans *pTrans) { + if (pTrans->prepareActions != NULL) { + mndTransDropActions(pTrans->prepareActions); + pTrans->prepareActions = NULL; + } if (pTrans->redoActions != NULL) { mndTransDropActions(pTrans->redoActions); pTrans->redoActions = NULL; @@ -549,7 +479,7 @@ static void mndTransDropData(STrans *pTrans) { (void)taosThreadMutexDestroy(&pTrans->mutex); } -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { +static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { mInfo("trans:%d, perform delete action, row:%p stage:%s callfunc:%d, stopFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), callFunc, pTrans->stopFunc); @@ -590,6 +520,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { return -1; } + mndTransUpdateActions(pOld->prepareActions, pNew->prepareActions); mndTransUpdateActions(pOld->redoActions, pNew->redoActions); mndTransUpdateActions(pOld->undoActions, pNew->undoActions); mndTransUpdateActions(pOld->commitActions, pNew->commitActions); @@ -646,6 +577,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->conflict = conflict; pTrans->exec = TRN_EXEC_PARALLEL; pTrans->createdTime = taosGetTimestampMs(); + pTrans->prepareActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); @@ -728,6 +660,13 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { return mndTransAppendAction(pTrans->commitActions, &action); } +int32_t mndTransAppendPrepareAction(STrans *pTrans, STransAction *pAction) { + pAction->stage = TRN_STAGE_PREPARE; + pAction->actionType = TRANS_ACTION_RAW; + pAction->mTraceId = pTrans->mTraceId; + return mndTransAppendAction(pTrans->prepareActions, pAction); +} + int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_REDO_ACTION; pAction->actionType = TRANS_ACTION_MSG; @@ -800,7 +739,7 @@ void mndTransSetParallel(STrans *pTrans) { pTrans->exec = TRN_EXEC_PARALLEL; } void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; } static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { - SSdbRaw *pRaw = mndTransActionEncode(pTrans); + SSdbRaw *pRaw = mndTransEncode(pTrans); if (pRaw == NULL) { mError("trans:%d, failed to encode while sync trans since %s", pTrans->id, terrstr()); return -1; @@ -922,7 +861,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; - mndTransExecute(pMnode, pNew, true); + mndTransExecute(pMnode, pNew); mndReleaseTrans(pMnode, pNew); return 0; } @@ -1104,7 +1043,7 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code); } - mndTransExecute(pMnode, pTrans, true); + mndTransExecute(pMnode, pTrans); _OVER: mndReleaseTrans(pMnode, pTrans); @@ -1550,7 +1489,7 @@ static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = false; - SSdbRaw *pRaw = mndTransActionEncode(pTrans); + SSdbRaw *pRaw = mndTransEncode(pTrans); if (pRaw == NULL) { mError("trans:%d, failed to encode while finish trans since %s", pTrans->id, terrstr()); return false; @@ -1567,12 +1506,12 @@ static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { +void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; while (continueExec) { - mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " leader:%d", pTrans->id, - mndTransStr(pTrans->stage), pTrans->createdTime, isLeader); + mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf:%d", pTrans->id, + mndTransStr(pTrans->stage), pTrans->createdTime, topHalf); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: @@ -1582,7 +1521,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = mndTransPerformRedoActionStage(pMnode, pTrans); break; case TRN_STAGE_COMMIT: - if (isLeader) { + if (topHalf) { continueExec = mndTransPerformCommitStage(pMnode, pTrans); } else { mInfo("trans:%d, can not commit since not leader", pTrans->id); @@ -1593,7 +1532,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = mndTransPerformCommitActionStage(pMnode, pTrans); break; case TRN_STAGE_ROLLBACK: - if (isLeader) { + if (topHalf) { continueExec = mndTransPerformRollbackStage(pMnode, pTrans); } else { mInfo("trans:%d, can not rollback since not leader", pTrans->id); @@ -1604,7 +1543,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = mndTransPerformUndoActionStage(pMnode, pTrans); break; case TRN_STAGE_PRE_FINISH: - if (isLeader) { + if (topHalf) { continueExec = mndTransPerformPreFinishStage(pMnode, pTrans); } else { mInfo("trans:%d, can not pre-finish since not leader", pTrans->id); @@ -1623,6 +1562,16 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { mndTransSendRpcRsp(pMnode, pTrans); } +void mndTransExecute(SMnode *pMnode, STrans *pTrans) { + bool topHalf = true; + return mndTransExecuteImp(pMnode, pTrans, topHalf); +} + +void mndTransRefresh(SMnode *pMnode, STrans *pTrans) { + bool topHalf = false; + return mndTransExecuteImp(pMnode, pTrans, topHalf); +} + static int32_t mndProcessTransTimer(SRpcMsg *pReq) { mTrace("start to process trans timer"); mndTransPullup(pReq->info.node); @@ -1649,7 +1598,7 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { pAction->errCode = 0; } - mndTransExecute(pMnode, pTrans, true); + mndTransExecute(pMnode, pTrans); return 0; } @@ -1707,7 +1656,7 @@ void mndTransPullup(SMnode *pMnode) { int32_t *pTransId = taosArrayGet(pArray, i); STrans *pTrans = mndAcquireTrans(pMnode, *pTransId); if (pTrans != NULL) { - mndTransExecute(pMnode, pTrans, true); + mndTransExecute(pMnode, pTrans); } mndReleaseTrans(pMnode, pTrans); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 7691f0f8f4..a5a364023d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2255,11 +2255,13 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro SDbObj dbObj = {0}; SArray *pArray = mndBuildDnodesArray(pMnode, 0); - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "split-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); + mndTransSetDbName(pTrans, pDb->name, NULL); + SVgObj newVg1 = {0}; memcpy(&newVg1, pVgroup, sizeof(SVgObj)); mInfo("vgId:%d, vgroup info before split, replica:%d hashBegin:%u hashEnd:%u", newVg1.vgId, newVg1.replica, From 3c698b24613a718cab8bc14e0e920cee5f328621 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 24 May 2023 18:25:56 +0800 Subject: [PATCH 009/122] enh: add mndAddPrepareNewVgAction --- source/dnode/mnode/impl/inc/mndTrans.h | 2 +- source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 10 ++++++- source/dnode/mnode/impl/src/mndSma.c | 4 +-- source/dnode/mnode/impl/src/mndTrans.c | 13 +++++++++ source/dnode/mnode/impl/src/mndVgroup.c | 35 +++++++++++++++++++------ source/dnode/mnode/sdb/inc/sdb.h | 1 + source/dnode/mnode/sdb/src/sdbHash.c | 1 + 8 files changed, 55 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 9fa59b4ac6..ac078ce1a3 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -70,7 +70,7 @@ int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendNullLog(STrans *pTrans); -int32_t mndTransAppendPrepareAction(STrans *pTrans, STransAction *pRaw); +int32_t mndTransAppendPrepareAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 0cd1228f25..286a624383 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -36,6 +36,7 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddPrepareNewVgAction(SMnode *, STrans *pTrans, SVgObj *pVg); int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid); int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 15374a8ddd..05acaea7bc 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -414,6 +414,13 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; } +static int32_t mndSetPrepareNewVgActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { + if (mndAddPrepareNewVgAction(pMnode, pTrans, (pVgroups + v)) != 0) return -1; + } + return 0; +} + static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); if (pDbRaw == NULL) return -1; @@ -424,7 +431,7 @@ static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); if (pVgRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE) != 0) return -1; } return 0; @@ -575,6 +582,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetOper(pTrans, MND_OPER_CREATE_DB); + if (mndSetPrepareNewVgActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 996f6c6494..c337d85b68 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -388,7 +388,7 @@ static int32_t mndSetCreateSmaVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVg SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); if (pVgRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE) != 0) return -1; return 0; } @@ -626,7 +626,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea mndTransSetSerial(pTrans); mInfo("trans:%d, used to create sma:%s stream:%s", pTrans->id, pCreate->name, streamObj.name); - + if (mndAddPrepareNewVgAction(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 69a6053ba6..7822a596d4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1333,6 +1333,19 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = true; + int32_t code = 0; + + int32_t numOfActions = taosArrayGetSize(pTrans->prepareActions); + if (numOfActions == 0) goto _OVER; + + mInfo("trans:%d, execute %d prepare actions.", pTrans->id, numOfActions); + + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); + mndTransExecSingleAction(pMnode, pTrans, pAction); + } + +_OVER: pTrans->stage = TRN_STAGE_REDO_ACTION; mInfo("trans:%d, stage from prepare to redoAction", pTrans->id); return continueExec; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a5a364023d..58612efe48 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -483,15 +483,15 @@ static void *mndBuildDisableVnodeWriteReq(SMnode *pMnode, SDbObj *pDb, int32_t v return pReq; } -static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dstVgId, int32_t *pContLen) { +static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, int32_t srcVgId, SVgObj *pVgroup, int32_t *pContLen) { SAlterVnodeHashRangeReq alterReq = { - .srcVgId = pVgroup->vgId, - .dstVgId = dstVgId, + .srcVgId = srcVgId, + .dstVgId = pVgroup->vgId, .hashBegin = pVgroup->hashBegin, .hashEnd = pVgroup->hashEnd, }; - mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", pVgroup->vgId, dstVgId, + mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", srcVgId, pVgroup->vgId, pVgroup->hashBegin, pVgroup->hashEnd); int32_t contLen = tSerializeSAlterVnodeHashRangeReq(NULL, 0, &alterReq); if (contLen < 0) { @@ -1207,12 +1207,12 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } -static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dstVgId) { +static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, int32_t srcVgId, SVgObj *pVgroup) { STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, pVgroup, dstVgId, &contLen); + void *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, srcVgId, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; @@ -1247,6 +1247,21 @@ int32_t mndAddAlterVnodeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb return 0; } +int32_t mndAddPrepareNewVgAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVg) { + SSdbRaw *pRaw = mndVgroupActionEncode(pVg); + if (pRaw == NULL) goto _err; + + STransAction action = {.pRaw = pRaw}; + if (mndTransAppendPrepareAction(pTrans, &action) != 0) goto _err; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); + pRaw = NULL; + return 0; + +_err: + sdbFreeRaw(pRaw); + return -1; +} + int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); if (pDnode == NULL) return -1; @@ -2313,12 +2328,16 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro // alter vgId and hash range int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg1, maxVgId) != 0) goto _OVER; + int32_t srcVgId = newVg1.vgId; newVg1.vgId = maxVgId; + if (mndAddPrepareNewVgAction(pMnode, pTrans, &newVg1) != 0) goto _OVER; + if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg1) != 0) goto _OVER; maxVgId++; - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg2, maxVgId) != 0) goto _OVER; + srcVgId = newVg2.vgId; newVg2.vgId = maxVgId; + if (mndAddPrepareNewVgAction(pMnode, pTrans, &newVg2) != 0) goto _OVER; + if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg2) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index e9a9e425e3..3c96d8a2fd 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -122,6 +122,7 @@ typedef enum { SDB_STATUS_DROPPING = 2, SDB_STATUS_DROPPED = 3, SDB_STATUS_READY = 4, + SDB_STATUS_UPDATE = 5, } ESdbStatus; typedef enum { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index f1cee6395b..258b22d8ee 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -256,6 +256,7 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) { code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); break; case SDB_STATUS_READY: + case SDB_STATUS_UPDATE: case SDB_STATUS_DROPPING: code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); break; From 57c301553bdb88785e9028dc809b1d99b6c0617d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 29 May 2023 16:40:30 +0800 Subject: [PATCH 010/122] feat(query): support single point interp syntax --- source/libs/parser/inc/parAst.h | 1 + source/libs/parser/inc/sql.y | 2 + source/libs/parser/src/parAstCreater.c | 5 + source/libs/parser/src/sql.c | 1566 ++++++++++++------------ 4 files changed, 794 insertions(+), 780 deletions(-) diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 43765c8112..e49d7b5a1d 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -127,6 +127,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); +SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9b8393f624..48e2b12b72 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1095,6 +1095,8 @@ having_clause_opt(A) ::= HAVING search_condition(B). range_opt(A) ::= . { A = NULL; } range_opt(A) ::= RANGE NK_LP expr_or_subquery(B) NK_COMMA expr_or_subquery(C) NK_RP. { A = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } +range_opt(A) ::= + RANGE NK_LP expr_or_subquery(B) NK_RP. { A = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, B)); } every_opt(A) ::= . { A = NULL; } every_opt(A) ::= EVERY NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 4223455b65..5ae6c85312 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -684,6 +684,11 @@ SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd); } +SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint) { + CHECK_PARSER_STATUS(pCxt); + return createOperatorNode(pCxt, OP_TYPE_EQUAL, createPrimaryKeyCol(pCxt, NULL), pPoint); +} + SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) { CHECK_PARSER_STATUS(pCxt); SWhenThenNode* pWhenThen = (SWhenThenNode*)nodesMakeNode(QUERY_NODE_WHEN_THEN); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 43ef166c41..ecf3b16672 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -141,17 +141,17 @@ typedef union { #define ParseCTX_STORE #define YYFALLBACK 1 #define YYNSTATE 792 -#define YYNRULE 595 -#define YYNRULE_WITH_ACTION 595 +#define YYNRULE 596 +#define YYNRULE_WITH_ACTION 596 #define YYNTOKEN 337 #define YY_MAX_SHIFT 791 -#define YY_MIN_SHIFTREDUCE 1169 -#define YY_MAX_SHIFTREDUCE 1763 -#define YY_ERROR_ACTION 1764 -#define YY_ACCEPT_ACTION 1765 -#define YY_NO_ACTION 1766 -#define YY_MIN_REDUCE 1767 -#define YY_MAX_REDUCE 2361 +#define YY_MIN_SHIFTREDUCE 1170 +#define YY_MAX_SHIFTREDUCE 1765 +#define YY_ERROR_ACTION 1766 +#define YY_ACCEPT_ACTION 1767 +#define YY_NO_ACTION 1768 +#define YY_MIN_REDUCE 1769 +#define YY_MAX_REDUCE 2364 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -220,278 +220,278 @@ typedef union { *********** Begin parsing tables **********************************************/ #define YY_ACTTAB_COUNT (2716) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2173, 169, 2337, 38, 303, 2332, 170, 445, 1779, 1885, - /* 10 */ 660, 444, 48, 46, 1691, 1933, 181, 2337, 2151, 1790, - /* 20 */ 399, 2336, 1540, 41, 40, 2333, 2335, 47, 45, 44, - /* 30 */ 43, 42, 2159, 1621, 1567, 1538, 366, 2046, 2191, 41, - /* 40 */ 40, 451, 2155, 47, 45, 44, 43, 42, 402, 663, - /* 50 */ 2141, 1568, 698, 1566, 620, 526, 164, 2332, 527, 1803, - /* 60 */ 2173, 218, 1616, 380, 1946, 529, 167, 1810, 19, 2141, - /* 70 */ 699, 1995, 2338, 188, 1947, 1546, 184, 2333, 646, 2157, - /* 80 */ 396, 2106, 2337, 358, 2172, 2332, 345, 2208, 1984, 692, - /* 90 */ 110, 2174, 702, 2176, 2177, 697, 620, 692, 2191, 2332, - /* 100 */ 788, 2336, 185, 15, 2261, 2333, 2334, 220, 395, 2257, - /* 110 */ 2141, 529, 698, 1810, 2338, 188, 48, 46, 1753, 2333, - /* 120 */ 646, 190, 682, 1944, 399, 1789, 1540, 1650, 251, 2287, - /* 130 */ 47, 45, 44, 43, 42, 668, 1567, 1621, 205, 1538, - /* 140 */ 1623, 1624, 133, 2063, 2172, 682, 1944, 2208, 1304, 566, - /* 150 */ 110, 2174, 702, 2176, 2177, 697, 2124, 692, 2061, 669, - /* 160 */ 145, 1565, 152, 2232, 2261, 193, 1616, 668, 395, 2257, - /* 170 */ 1596, 1606, 19, 1395, 1396, 2141, 1622, 1625, 84, 1546, - /* 180 */ 1920, 83, 657, 142, 1651, 1768, 284, 541, 1306, 2056, - /* 190 */ 1541, 123, 1539, 681, 122, 121, 120, 119, 118, 117, - /* 200 */ 116, 115, 114, 262, 788, 668, 123, 15, 2173, 122, + /* 0 */ 2175, 169, 2340, 38, 303, 2335, 170, 445, 1781, 1887, + /* 10 */ 660, 444, 48, 46, 1692, 1935, 181, 2340, 2153, 1792, + /* 20 */ 399, 2339, 1541, 41, 40, 2336, 2338, 47, 45, 44, + /* 30 */ 43, 42, 2161, 1622, 1568, 1539, 366, 2048, 2193, 41, + /* 40 */ 40, 451, 2157, 47, 45, 44, 43, 42, 402, 663, + /* 50 */ 2143, 1569, 698, 1567, 620, 526, 164, 2335, 527, 1805, + /* 60 */ 2175, 218, 1617, 380, 1948, 529, 167, 1812, 19, 2143, + /* 70 */ 699, 1997, 2341, 188, 1949, 1547, 184, 2336, 646, 2159, + /* 80 */ 396, 2108, 2340, 358, 2174, 2335, 345, 2210, 1986, 692, + /* 90 */ 110, 2176, 702, 2178, 2179, 697, 620, 692, 2193, 2335, + /* 100 */ 788, 2339, 185, 15, 2263, 2336, 2337, 220, 395, 2259, + /* 110 */ 2143, 529, 698, 1812, 2341, 188, 48, 46, 1755, 2336, + /* 120 */ 646, 190, 682, 1946, 399, 1791, 1541, 1651, 251, 2289, + /* 130 */ 47, 45, 44, 43, 42, 668, 1568, 1622, 205, 1539, + /* 140 */ 1624, 1625, 133, 2065, 2174, 682, 1946, 2210, 1305, 566, + /* 150 */ 110, 2176, 702, 2178, 2179, 697, 2129, 692, 2063, 669, + /* 160 */ 145, 1566, 152, 2234, 2263, 193, 1617, 668, 395, 2259, + /* 170 */ 1597, 1607, 19, 1396, 1397, 2143, 1623, 1626, 84, 1547, + /* 180 */ 1922, 83, 657, 142, 1652, 1770, 284, 541, 1307, 2058, + /* 190 */ 1542, 123, 1540, 681, 122, 121, 120, 119, 118, 117, + /* 200 */ 116, 115, 114, 262, 788, 421, 123, 15, 2175, 122, /* 210 */ 121, 120, 119, 118, 117, 116, 115, 114, 699, 666, - /* 220 */ 1812, 2056, 1544, 1545, 1767, 1595, 1598, 1599, 1600, 1601, - /* 230 */ 1602, 1603, 1604, 1605, 694, 690, 1614, 1615, 1617, 1618, - /* 240 */ 1619, 1620, 2, 681, 1623, 1624, 2191, 737, 132, 131, - /* 250 */ 130, 129, 128, 127, 126, 125, 124, 677, 2141, 2056, - /* 260 */ 698, 37, 397, 1645, 1646, 1647, 1648, 1649, 1653, 1654, - /* 270 */ 1655, 1656, 543, 2173, 1596, 1606, 51, 682, 1944, 66, - /* 280 */ 1622, 1625, 9, 660, 659, 186, 2269, 2270, 284, 140, - /* 290 */ 2274, 1227, 2172, 1226, 1541, 2208, 1539, 133, 110, 2174, - /* 300 */ 702, 2176, 2177, 697, 571, 692, 138, 531, 682, 1944, - /* 310 */ 2352, 2191, 2261, 528, 41, 40, 395, 2257, 47, 45, - /* 320 */ 44, 43, 42, 2141, 1228, 698, 1544, 1545, 57, 1595, - /* 330 */ 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 694, 690, - /* 340 */ 1614, 1615, 1617, 1618, 1619, 1620, 2, 12, 48, 46, - /* 350 */ 1227, 2173, 1226, 408, 407, 2336, 399, 2172, 1540, 2276, - /* 360 */ 2208, 696, 1922, 110, 2174, 702, 2176, 2177, 697, 1621, - /* 370 */ 692, 1538, 428, 1997, 181, 185, 1547, 2261, 62, 639, - /* 380 */ 379, 395, 2257, 1228, 2063, 2273, 41, 40, 1995, 2191, - /* 390 */ 47, 45, 44, 43, 42, 2047, 392, 1360, 1616, 2060, - /* 400 */ 669, 2141, 2288, 698, 19, 534, 150, 2191, 527, 1803, - /* 410 */ 1730, 1546, 1351, 727, 726, 725, 1355, 724, 1357, 1358, - /* 420 */ 723, 720, 30, 1366, 717, 1368, 1369, 714, 711, 708, - /* 430 */ 667, 1695, 191, 1565, 107, 2172, 788, 1565, 2208, 15, - /* 440 */ 101, 339, 2174, 702, 2176, 2177, 697, 695, 692, 683, - /* 450 */ 2226, 143, 48, 46, 1626, 87, 635, 34, 234, 1936, - /* 460 */ 399, 638, 1540, 41, 40, 1937, 681, 47, 45, 44, - /* 470 */ 43, 42, 368, 1621, 174, 1538, 1623, 1624, 640, 682, - /* 480 */ 1944, 1939, 560, 556, 552, 548, 2042, 233, 285, 393, - /* 490 */ 735, 157, 156, 732, 731, 730, 154, 167, 2173, 449, - /* 500 */ 1202, 1684, 1616, 1565, 1719, 1946, 1596, 1606, 699, 12, - /* 510 */ 2295, 10, 1622, 1625, 2042, 1546, 41, 40, 682, 1944, - /* 520 */ 47, 45, 44, 43, 42, 1564, 1541, 88, 1539, 62, - /* 530 */ 231, 93, 201, 1550, 191, 489, 2191, 1568, 450, 1204, - /* 540 */ 788, 1207, 1208, 49, 1788, 641, 636, 629, 2141, 51, - /* 550 */ 698, 632, 631, 1717, 1718, 1720, 1721, 1722, 1544, 1545, - /* 560 */ 203, 1595, 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, - /* 570 */ 694, 690, 1614, 1615, 1617, 1618, 1619, 1620, 2, 62, - /* 580 */ 1623, 1624, 2172, 1450, 1451, 2208, 62, 1597, 110, 2174, - /* 590 */ 702, 2176, 2177, 697, 2141, 692, 209, 208, 230, 224, - /* 600 */ 2352, 523, 2261, 229, 737, 539, 395, 2257, 2173, 521, - /* 610 */ 1596, 1606, 517, 513, 682, 1944, 1622, 1625, 699, 488, - /* 620 */ 2308, 682, 1944, 222, 607, 2151, 52, 62, 294, 295, - /* 630 */ 1541, 648, 1539, 293, 459, 584, 583, 582, 684, 1935, - /* 640 */ 2233, 474, 574, 139, 578, 2127, 2191, 1787, 577, 2155, - /* 650 */ 657, 142, 1549, 576, 581, 374, 373, 2027, 2141, 575, - /* 660 */ 698, 2134, 1544, 1545, 1566, 1595, 1598, 1599, 1600, 1601, - /* 670 */ 1602, 1603, 1604, 1605, 694, 690, 1614, 1615, 1617, 1618, - /* 680 */ 1619, 1620, 2, 48, 46, 191, 2157, 682, 1944, 202, - /* 690 */ 1760, 399, 2172, 1540, 421, 2208, 692, 2141, 110, 2174, - /* 700 */ 702, 2176, 2177, 697, 1621, 692, 1538, 475, 620, 2173, - /* 710 */ 2352, 2332, 2261, 443, 402, 442, 395, 2257, 1921, 699, - /* 720 */ 404, 589, 167, 1990, 1992, 2276, 2338, 188, 682, 1944, - /* 730 */ 1946, 2333, 646, 1616, 1631, 191, 599, 1991, 1992, 2173, - /* 740 */ 1565, 433, 191, 682, 1944, 441, 1546, 2191, 542, 699, - /* 750 */ 247, 2272, 598, 282, 2269, 656, 12, 134, 655, 2141, - /* 760 */ 2332, 698, 405, 1941, 2276, 596, 592, 594, 435, 431, - /* 770 */ 167, 788, 1546, 586, 49, 644, 188, 2191, 1946, 246, - /* 780 */ 2333, 646, 191, 191, 1467, 1468, 747, 48, 46, 2141, - /* 790 */ 2271, 698, 1759, 2172, 1315, 399, 2208, 1540, 250, 111, - /* 800 */ 2174, 702, 2176, 2177, 697, 1652, 692, 1314, 1621, 1552, - /* 810 */ 1538, 1623, 1624, 2261, 657, 142, 469, 2260, 2257, 70, - /* 820 */ 1466, 1469, 69, 2172, 1919, 468, 2208, 728, 1997, 171, - /* 830 */ 2174, 702, 2176, 2177, 697, 389, 692, 1616, 1786, 2173, - /* 840 */ 1210, 1596, 1606, 1995, 682, 1944, 1564, 1622, 1625, 699, - /* 850 */ 1546, 735, 157, 156, 732, 731, 730, 154, 452, 1997, - /* 860 */ 686, 1541, 2233, 1539, 252, 155, 394, 1511, 1512, 621, - /* 870 */ 2298, 453, 498, 36, 1995, 788, 1929, 2191, 15, 41, - /* 880 */ 40, 1931, 35, 47, 45, 44, 43, 42, 2141, 2141, - /* 890 */ 1597, 698, 1657, 1544, 1545, 1927, 1595, 1598, 1599, 1600, - /* 900 */ 1601, 1602, 1603, 1604, 1605, 694, 690, 1614, 1615, 1617, - /* 910 */ 1618, 1619, 1620, 2, 1785, 1623, 1624, 187, 2269, 2270, - /* 920 */ 1765, 140, 2274, 2172, 1784, 56, 2208, 1783, 1782, 171, - /* 930 */ 2174, 702, 2176, 2177, 697, 349, 692, 1563, 44, 43, - /* 940 */ 42, 1997, 259, 2173, 482, 1596, 1606, 496, 403, 1781, - /* 950 */ 495, 1622, 1625, 699, 1997, 627, 1995, 735, 157, 156, - /* 960 */ 732, 731, 730, 154, 2141, 1541, 465, 1539, 497, 1996, - /* 970 */ 2299, 682, 1944, 467, 2141, 1540, 1319, 2141, 2141, 41, - /* 980 */ 40, 2191, 1778, 47, 45, 44, 43, 42, 1538, 1318, - /* 990 */ 1568, 616, 414, 2141, 1777, 698, 413, 1544, 1545, 2141, - /* 1000 */ 1595, 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 694, - /* 1010 */ 690, 1614, 1615, 1617, 1618, 1619, 1620, 2, 1843, 367, - /* 1020 */ 1565, 682, 1944, 657, 142, 580, 579, 2172, 1546, 1948, - /* 1030 */ 2208, 455, 2141, 110, 2174, 702, 2176, 2177, 697, 620, - /* 1040 */ 692, 661, 2332, 1776, 2141, 2352, 570, 2261, 1230, 1231, - /* 1050 */ 569, 395, 2257, 788, 500, 562, 561, 2338, 188, 564, - /* 1060 */ 563, 493, 2333, 646, 487, 486, 485, 484, 481, 480, + /* 220 */ 1814, 2058, 1545, 1546, 1769, 1596, 1599, 1600, 1601, 1602, + /* 230 */ 1603, 1604, 1605, 1606, 694, 690, 1615, 1616, 1618, 1619, + /* 240 */ 1620, 1621, 2, 681, 1624, 1625, 2193, 737, 132, 131, + /* 250 */ 130, 129, 128, 127, 126, 125, 124, 1923, 2143, 452, + /* 260 */ 698, 37, 397, 1646, 1647, 1648, 1649, 1650, 1654, 1655, + /* 270 */ 1656, 1657, 453, 2175, 1597, 1607, 51, 682, 1946, 66, + /* 280 */ 1623, 1626, 1569, 660, 659, 186, 2271, 2272, 284, 140, + /* 290 */ 2276, 1228, 2174, 1227, 1542, 2210, 1540, 133, 110, 2176, + /* 300 */ 702, 2178, 2179, 697, 571, 692, 138, 531, 682, 1946, + /* 310 */ 2355, 2193, 2263, 528, 41, 40, 395, 2259, 47, 45, + /* 320 */ 44, 43, 42, 2143, 1229, 698, 1545, 1546, 57, 1596, + /* 330 */ 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1606, 694, 690, + /* 340 */ 1615, 1616, 1618, 1619, 1620, 1621, 2, 12, 48, 46, + /* 350 */ 30, 2175, 144, 408, 407, 2234, 399, 2174, 1541, 729, + /* 360 */ 2210, 696, 1990, 110, 2176, 702, 2178, 2179, 697, 1622, + /* 370 */ 692, 1539, 433, 34, 2164, 185, 1548, 2263, 62, 41, + /* 380 */ 40, 395, 2259, 47, 45, 44, 43, 42, 62, 2193, + /* 390 */ 735, 157, 156, 732, 731, 730, 154, 1361, 1617, 435, + /* 400 */ 431, 2143, 2290, 698, 19, 534, 1566, 668, 527, 1805, + /* 410 */ 1732, 1547, 1352, 727, 726, 725, 1356, 724, 1358, 1359, + /* 420 */ 723, 720, 681, 1367, 717, 1369, 1370, 714, 711, 708, + /* 430 */ 2166, 1696, 191, 1566, 202, 2174, 788, 1566, 2210, 15, + /* 440 */ 52, 339, 2176, 702, 2178, 2179, 697, 695, 692, 683, + /* 450 */ 2228, 1790, 48, 46, 1627, 87, 635, 36, 234, 677, + /* 460 */ 399, 2058, 1541, 41, 40, 181, 667, 47, 45, 44, + /* 470 */ 43, 42, 368, 1622, 174, 1539, 1624, 1625, 101, 682, + /* 480 */ 1946, 1941, 560, 556, 552, 548, 2049, 233, 285, 62, + /* 490 */ 735, 157, 156, 732, 731, 730, 154, 543, 2175, 449, + /* 500 */ 1203, 2143, 1617, 1939, 1720, 51, 1597, 1607, 699, 2278, + /* 510 */ 2297, 87, 1623, 1626, 2044, 1547, 41, 40, 682, 1946, + /* 520 */ 47, 45, 44, 43, 42, 737, 1542, 88, 1540, 62, + /* 530 */ 231, 93, 640, 1551, 191, 2275, 2193, 1942, 450, 1205, + /* 540 */ 788, 1208, 1209, 49, 191, 641, 636, 629, 2143, 1565, + /* 550 */ 698, 632, 631, 1718, 1719, 1721, 1722, 1723, 1545, 1546, + /* 560 */ 201, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1606, + /* 570 */ 694, 690, 1615, 1616, 1618, 1619, 1620, 1621, 2, 1999, + /* 580 */ 1624, 1625, 2174, 1451, 1452, 2210, 379, 1598, 110, 2176, + /* 590 */ 702, 2178, 2179, 697, 1997, 692, 191, 2065, 230, 224, + /* 600 */ 2355, 607, 2263, 229, 1999, 539, 395, 2259, 2175, 392, + /* 610 */ 1597, 1607, 2062, 669, 682, 1946, 1623, 1626, 699, 1998, + /* 620 */ 2310, 682, 1946, 222, 12, 2153, 2278, 62, 294, 295, + /* 630 */ 1542, 1547, 1540, 293, 459, 584, 583, 582, 12, 1937, + /* 640 */ 10, 474, 574, 139, 578, 191, 2193, 1789, 577, 2157, + /* 650 */ 657, 142, 2274, 576, 581, 374, 373, 1653, 2143, 575, + /* 660 */ 698, 2136, 1545, 1546, 1567, 1596, 1599, 1600, 1601, 1602, + /* 670 */ 1603, 1604, 1605, 1606, 694, 690, 1615, 1616, 1618, 1619, + /* 680 */ 1620, 1621, 2, 48, 46, 191, 2159, 682, 1946, 2044, + /* 690 */ 1762, 399, 2174, 1541, 728, 2210, 692, 2143, 110, 2176, + /* 700 */ 702, 2178, 2179, 697, 1622, 692, 1539, 475, 620, 2175, + /* 710 */ 2355, 2335, 2263, 443, 393, 442, 395, 2259, 1921, 699, + /* 720 */ 404, 589, 167, 1992, 1994, 317, 2341, 188, 1976, 523, + /* 730 */ 1948, 2336, 646, 1617, 35, 203, 599, 521, 1316, 2175, + /* 740 */ 517, 513, 682, 1946, 1658, 441, 1547, 2193, 402, 699, + /* 750 */ 247, 1315, 747, 282, 2271, 656, 167, 134, 655, 2143, + /* 760 */ 2335, 698, 542, 1999, 1948, 107, 592, 1993, 1994, 733, + /* 770 */ 389, 788, 1990, 586, 49, 644, 188, 2193, 1997, 246, + /* 780 */ 2336, 646, 143, 191, 1468, 1469, 1931, 48, 46, 2143, + /* 790 */ 1938, 698, 1761, 2174, 1933, 399, 2210, 1541, 489, 111, + /* 800 */ 2176, 702, 2178, 2179, 697, 1228, 692, 1227, 1622, 2278, + /* 810 */ 1539, 1624, 1625, 2263, 657, 142, 498, 2262, 2259, 70, + /* 820 */ 1467, 1470, 69, 2174, 682, 1946, 2210, 562, 561, 171, + /* 830 */ 2176, 702, 2178, 2179, 697, 2273, 692, 1617, 1229, 2175, + /* 840 */ 1788, 1597, 1607, 639, 1943, 14, 13, 1623, 1626, 699, + /* 850 */ 1547, 735, 157, 156, 732, 731, 730, 154, 1999, 209, + /* 860 */ 208, 1542, 684, 1540, 2235, 394, 44, 43, 42, 621, + /* 870 */ 2300, 2193, 570, 1997, 1787, 788, 569, 2193, 15, 41, + /* 880 */ 40, 1929, 488, 47, 45, 44, 43, 42, 1786, 2143, + /* 890 */ 2143, 698, 2137, 1545, 1546, 1950, 1596, 1599, 1600, 1601, + /* 900 */ 1602, 1603, 1604, 1605, 1606, 694, 690, 1615, 1616, 1618, + /* 910 */ 1619, 1620, 1621, 2, 605, 1624, 1625, 187, 2271, 2272, + /* 920 */ 1767, 140, 2276, 2174, 2143, 638, 2210, 1785, 1784, 171, + /* 930 */ 2176, 702, 2178, 2179, 697, 349, 692, 1564, 2143, 620, + /* 940 */ 682, 1946, 2335, 2175, 482, 1597, 1607, 496, 1665, 1783, + /* 950 */ 495, 1623, 1626, 699, 1689, 627, 250, 2341, 188, 259, + /* 960 */ 252, 620, 2336, 646, 2335, 1542, 465, 1540, 497, 256, + /* 970 */ 2301, 682, 1946, 467, 405, 1541, 1320, 2143, 2143, 2341, + /* 980 */ 188, 2193, 167, 2044, 2336, 646, 1231, 1232, 1539, 1319, + /* 990 */ 1948, 616, 414, 2143, 1780, 698, 413, 1545, 1546, 2143, + /* 1000 */ 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1606, 694, + /* 1010 */ 690, 1615, 1616, 1618, 1619, 1620, 1621, 2, 1845, 367, + /* 1020 */ 2126, 682, 1946, 657, 142, 1512, 1513, 2174, 1547, 207, + /* 1030 */ 2210, 455, 9, 110, 2176, 702, 2178, 2179, 697, 620, + /* 1040 */ 692, 661, 2335, 166, 2143, 2355, 54, 2263, 3, 1632, + /* 1050 */ 2029, 395, 2259, 788, 500, 1566, 649, 2341, 188, 564, + /* 1060 */ 563, 493, 2336, 646, 487, 486, 485, 484, 481, 480, /* 1070 */ 479, 478, 477, 473, 472, 471, 470, 348, 462, 461, - /* 1080 */ 460, 652, 457, 456, 365, 759, 757, 1775, 765, 764, - /* 1090 */ 763, 762, 411, 2141, 761, 760, 146, 755, 754, 753, + /* 1080 */ 460, 693, 457, 456, 365, 580, 579, 1779, 765, 764, + /* 1090 */ 763, 762, 411, 2339, 761, 760, 146, 755, 754, 753, /* 1100 */ 752, 751, 750, 749, 159, 745, 744, 743, 410, 409, - /* 1110 */ 740, 739, 738, 177, 176, 168, 682, 1944, 2135, 1774, - /* 1120 */ 323, 1773, 1772, 408, 407, 256, 189, 2269, 2270, 2173, - /* 1130 */ 140, 2274, 371, 1554, 321, 73, 665, 2141, 72, 699, - /* 1140 */ 693, 2326, 261, 1541, 1621, 1539, 1547, 41, 40, 346, - /* 1150 */ 1688, 47, 45, 44, 43, 42, 2042, 1771, 682, 1944, - /* 1160 */ 216, 508, 506, 503, 54, 620, 3, 2191, 2332, 2141, - /* 1170 */ 1597, 2141, 2141, 1616, 649, 1544, 1545, 2173, 298, 2141, - /* 1180 */ 1770, 698, 260, 2338, 188, 2151, 1546, 699, 2333, 646, - /* 1190 */ 41, 40, 14, 13, 47, 45, 44, 43, 42, 2160, - /* 1200 */ 62, 372, 207, 370, 369, 74, 568, 2141, 87, 2155, - /* 1210 */ 689, 688, 605, 2172, 144, 2191, 2208, 2232, 1886, 110, - /* 1220 */ 2174, 702, 2176, 2177, 697, 91, 692, 2141, 570, 698, - /* 1230 */ 2141, 2352, 569, 2261, 1940, 2173, 572, 395, 2257, 109, - /* 1240 */ 148, 729, 135, 1707, 1988, 699, 2157, 2280, 733, 734, - /* 1250 */ 317, 1988, 1988, 1974, 748, 82, 692, 1906, 1302, 620, - /* 1260 */ 239, 2172, 2332, 237, 2208, 2281, 1684, 333, 2174, 702, - /* 1270 */ 2176, 2177, 697, 2191, 692, 1207, 1208, 2338, 188, 81, - /* 1280 */ 80, 448, 2333, 646, 200, 2141, 2173, 698, 241, 1780, - /* 1290 */ 573, 240, 619, 682, 1944, 1664, 699, 440, 438, 243, - /* 1300 */ 2301, 1555, 242, 1550, 245, 1762, 1763, 244, 347, 1830, - /* 1310 */ 642, 429, 1300, 679, 427, 423, 419, 416, 441, 2172, - /* 1320 */ 653, 602, 2208, 601, 2191, 110, 2174, 702, 2176, 2177, - /* 1330 */ 697, 585, 692, 1558, 1560, 1821, 2141, 2352, 698, 2261, - /* 1340 */ 1819, 155, 645, 395, 2257, 2332, 690, 1614, 1615, 1617, - /* 1350 */ 1618, 1619, 1620, 2173, 166, 249, 191, 587, 155, 248, - /* 1360 */ 644, 188, 590, 699, 1642, 2333, 646, 279, 50, 50, - /* 1370 */ 2172, 266, 155, 2208, 50, 291, 110, 2174, 702, 2176, - /* 1380 */ 2177, 697, 633, 692, 682, 1944, 2173, 137, 2236, 106, - /* 1390 */ 2261, 2191, 682, 1944, 395, 2257, 699, 682, 1944, 103, - /* 1400 */ 71, 1506, 2162, 2141, 680, 698, 1687, 153, 155, 14, - /* 1410 */ 13, 650, 304, 2173, 64, 273, 55, 406, 1509, 90, - /* 1420 */ 1548, 1813, 353, 699, 2191, 378, 50, 600, 1716, 1715, - /* 1430 */ 741, 268, 664, 50, 1464, 296, 2141, 2172, 698, 2192, - /* 1440 */ 2208, 1884, 706, 110, 2174, 702, 2176, 2177, 697, 2173, - /* 1450 */ 692, 2191, 1280, 1883, 153, 2234, 155, 2261, 2164, 699, - /* 1460 */ 674, 395, 2257, 2141, 136, 698, 412, 300, 1345, 153, - /* 1470 */ 2172, 783, 1261, 2208, 1658, 2173, 110, 2174, 702, 2176, - /* 1480 */ 2177, 697, 2051, 692, 1804, 699, 1607, 2191, 685, 742, - /* 1490 */ 2261, 1809, 1985, 316, 395, 2257, 2291, 2172, 658, 2141, - /* 1500 */ 2208, 698, 1373, 111, 2174, 702, 2176, 2177, 697, 281, - /* 1510 */ 692, 1278, 1262, 2191, 1377, 278, 1384, 2261, 1, 5, - /* 1520 */ 415, 687, 2257, 420, 1382, 2141, 362, 698, 1571, 158, - /* 1530 */ 437, 436, 196, 700, 439, 198, 2208, 195, 1487, 111, - /* 1540 */ 2174, 702, 2176, 2177, 697, 311, 692, 206, 454, 1568, - /* 1550 */ 2052, 458, 491, 2261, 2173, 463, 1563, 357, 2257, 2172, - /* 1560 */ 476, 2044, 2208, 483, 699, 172, 2174, 702, 2176, 2177, - /* 1570 */ 697, 490, 692, 492, 501, 502, 499, 1551, 210, 504, - /* 1580 */ 2173, 211, 505, 213, 507, 509, 1569, 524, 4, 525, - /* 1590 */ 699, 532, 2191, 533, 1566, 221, 536, 535, 1570, 223, - /* 1600 */ 1572, 537, 565, 538, 2141, 1997, 698, 540, 544, 226, - /* 1610 */ 228, 85, 364, 86, 2115, 232, 647, 2353, 2191, 112, - /* 1620 */ 1995, 2112, 352, 382, 567, 604, 1934, 606, 2111, 89, - /* 1630 */ 2141, 610, 698, 609, 312, 151, 611, 253, 2172, 236, - /* 1640 */ 1930, 2208, 2173, 238, 111, 2174, 702, 2176, 2177, 697, - /* 1650 */ 160, 692, 699, 161, 1932, 255, 1928, 162, 2261, 2173, - /* 1660 */ 163, 614, 257, 2258, 2172, 1494, 617, 2208, 2292, 699, - /* 1670 */ 340, 2174, 702, 2176, 2177, 697, 672, 692, 2173, 615, - /* 1680 */ 2191, 624, 634, 645, 2302, 383, 2332, 8, 699, 630, - /* 1690 */ 2307, 2306, 2141, 264, 698, 267, 2283, 2191, 384, 643, - /* 1700 */ 637, 644, 188, 385, 625, 623, 2333, 646, 622, 2141, - /* 1710 */ 1684, 698, 654, 277, 651, 141, 2191, 2355, 1567, 175, - /* 1720 */ 2331, 390, 2277, 662, 388, 286, 2172, 274, 2141, 2208, - /* 1730 */ 698, 96, 340, 2174, 702, 2176, 2177, 697, 272, 692, - /* 1740 */ 1573, 313, 2057, 2172, 280, 276, 2208, 670, 275, 172, - /* 1750 */ 2174, 702, 2176, 2177, 697, 675, 692, 671, 2071, 314, - /* 1760 */ 676, 2070, 2172, 98, 2173, 2208, 2069, 391, 340, 2174, - /* 1770 */ 702, 2176, 2177, 697, 696, 692, 315, 1945, 100, 61, - /* 1780 */ 2242, 2173, 102, 704, 1989, 318, 784, 1907, 307, 322, - /* 1790 */ 785, 699, 787, 53, 354, 327, 355, 342, 2133, 320, - /* 1800 */ 2173, 2354, 2191, 341, 2132, 331, 2131, 78, 2128, 417, - /* 1810 */ 699, 418, 1531, 1532, 2141, 194, 698, 422, 2126, 2191, - /* 1820 */ 424, 425, 426, 2125, 398, 363, 2123, 430, 2122, 432, - /* 1830 */ 2121, 2141, 434, 698, 2102, 1522, 197, 2101, 2191, 199, - /* 1840 */ 1490, 79, 1489, 400, 2083, 2082, 2081, 446, 2172, 447, - /* 1850 */ 2141, 2208, 698, 2080, 339, 2174, 702, 2176, 2177, 697, - /* 1860 */ 2079, 692, 1441, 2227, 2035, 2172, 147, 608, 2208, 2034, - /* 1870 */ 2032, 340, 2174, 702, 2176, 2177, 697, 2031, 692, 2030, - /* 1880 */ 2033, 2029, 2028, 2026, 2172, 791, 2025, 2208, 2024, 204, - /* 1890 */ 340, 2174, 702, 2176, 2177, 697, 464, 692, 2173, 310, - /* 1900 */ 2023, 466, 2037, 2022, 2021, 2020, 2019, 2018, 699, 2017, - /* 1910 */ 2016, 2015, 2014, 2013, 2012, 180, 149, 2007, 2006, 2005, - /* 1920 */ 2036, 2004, 2003, 781, 777, 773, 769, 2011, 308, 2010, - /* 1930 */ 2009, 2173, 2008, 2002, 2001, 2000, 2191, 1443, 494, 1999, - /* 1940 */ 1998, 699, 350, 351, 1849, 1316, 1320, 212, 2141, 1848, - /* 1950 */ 698, 214, 1847, 215, 1845, 1842, 1841, 511, 1834, 1823, - /* 1960 */ 1312, 510, 1799, 1798, 2173, 227, 514, 512, 108, 2191, - /* 1970 */ 2100, 301, 2090, 2078, 699, 516, 518, 522, 2077, 1209, - /* 1980 */ 2055, 2141, 603, 698, 1923, 2208, 520, 515, 335, 2174, - /* 1990 */ 702, 2176, 2177, 697, 519, 692, 1844, 182, 2161, 217, - /* 2000 */ 2173, 1840, 2191, 77, 678, 547, 76, 183, 530, 219, - /* 2010 */ 699, 545, 1254, 225, 2141, 2172, 698, 546, 2208, 1838, - /* 2020 */ 549, 324, 2174, 702, 2176, 2177, 697, 551, 692, 1836, - /* 2030 */ 550, 554, 553, 2173, 555, 1833, 558, 557, 2191, 288, - /* 2040 */ 1818, 1816, 559, 699, 287, 1817, 1815, 1795, 2172, 1925, - /* 2050 */ 2141, 2208, 698, 1389, 325, 2174, 702, 2176, 2177, 697, - /* 2060 */ 63, 692, 2173, 1924, 254, 1388, 1303, 1301, 1299, 1298, - /* 2070 */ 756, 2191, 699, 1297, 1296, 1831, 1290, 1295, 375, 1292, - /* 2080 */ 758, 235, 1822, 2141, 2172, 698, 1291, 2208, 376, 1820, - /* 2090 */ 326, 2174, 702, 2176, 2177, 697, 1289, 692, 377, 588, - /* 2100 */ 2191, 597, 591, 1794, 593, 1793, 595, 1792, 113, 1516, - /* 2110 */ 1520, 29, 2141, 1518, 698, 1515, 2099, 2172, 58, 67, - /* 2120 */ 2208, 1496, 2173, 332, 2174, 702, 2176, 2177, 697, 2089, - /* 2130 */ 692, 1498, 699, 612, 2076, 2074, 2337, 20, 17, 6, - /* 2140 */ 31, 2173, 7, 21, 22, 271, 2172, 1500, 270, 2208, - /* 2150 */ 33, 699, 336, 2174, 702, 2176, 2177, 697, 613, 692, - /* 2160 */ 2191, 1732, 381, 258, 265, 263, 626, 628, 165, 2162, - /* 2170 */ 618, 65, 2141, 173, 698, 24, 1714, 1706, 1747, 2191, - /* 2180 */ 269, 1746, 386, 1751, 32, 1750, 92, 283, 387, 2173, - /* 2190 */ 2075, 2141, 178, 698, 1752, 60, 1753, 2073, 2072, 699, - /* 2200 */ 1681, 1680, 2054, 95, 289, 94, 2172, 2053, 2173, 2208, - /* 2210 */ 97, 25, 328, 2174, 702, 2176, 2177, 697, 699, 692, - /* 2220 */ 26, 290, 1712, 292, 297, 2172, 68, 2191, 2208, 99, - /* 2230 */ 673, 337, 2174, 702, 2176, 2177, 697, 299, 692, 2141, - /* 2240 */ 23, 698, 1846, 103, 11, 1633, 2191, 13, 302, 1556, - /* 2250 */ 59, 179, 1643, 1611, 1588, 18, 2173, 1632, 2141, 1609, - /* 2260 */ 698, 2211, 691, 39, 192, 1608, 699, 1580, 16, 27, - /* 2270 */ 28, 705, 401, 2172, 701, 707, 2208, 2173, 709, 329, - /* 2280 */ 2174, 702, 2176, 2177, 697, 1374, 692, 699, 703, 1371, - /* 2290 */ 710, 712, 2172, 1370, 2191, 2208, 713, 715, 338, 2174, - /* 2300 */ 702, 2176, 2177, 697, 718, 692, 2141, 1367, 698, 716, - /* 2310 */ 721, 719, 584, 583, 582, 2191, 1361, 1359, 722, 574, - /* 2320 */ 139, 578, 104, 305, 1365, 577, 105, 2141, 1383, 698, - /* 2330 */ 576, 581, 374, 373, 1364, 1363, 575, 75, 1379, 1362, - /* 2340 */ 2172, 1252, 736, 2208, 1284, 2173, 330, 2174, 702, 2176, - /* 2350 */ 2177, 697, 1283, 692, 1282, 699, 1281, 1279, 1277, 1276, - /* 2360 */ 1275, 2172, 1310, 2173, 2208, 746, 1270, 343, 2174, 702, - /* 2370 */ 2176, 2177, 697, 699, 692, 306, 1273, 1272, 1271, 1307, - /* 2380 */ 1269, 1268, 1267, 2191, 1305, 1264, 1263, 1260, 1259, 1258, - /* 2390 */ 1257, 1839, 766, 2173, 1837, 2141, 770, 698, 768, 772, - /* 2400 */ 1835, 2191, 776, 699, 774, 767, 1832, 771, 778, 780, - /* 2410 */ 1814, 775, 782, 2141, 779, 698, 1199, 1791, 309, 786, - /* 2420 */ 1766, 1542, 790, 319, 2173, 789, 1766, 1766, 1766, 2172, - /* 2430 */ 1766, 2191, 2208, 1766, 699, 344, 2174, 702, 2176, 2177, - /* 2440 */ 697, 2173, 692, 2141, 1766, 698, 1766, 2172, 1766, 1766, - /* 2450 */ 2208, 699, 1766, 2185, 2174, 702, 2176, 2177, 697, 1766, - /* 2460 */ 692, 1766, 2191, 1766, 1766, 1766, 1766, 1766, 1766, 1766, - /* 2470 */ 1766, 1766, 1766, 1766, 2141, 1766, 698, 2172, 1766, 2191, - /* 2480 */ 2208, 1766, 1766, 2184, 2174, 702, 2176, 2177, 697, 1766, - /* 2490 */ 692, 2141, 2173, 698, 1766, 1766, 1766, 1766, 1766, 1766, - /* 2500 */ 1766, 1766, 699, 1766, 1766, 1766, 1766, 1766, 2172, 1766, - /* 2510 */ 1766, 2208, 1766, 2173, 2183, 2174, 702, 2176, 2177, 697, - /* 2520 */ 1766, 692, 1766, 699, 1766, 2172, 1766, 1766, 2208, 1766, - /* 2530 */ 2191, 359, 2174, 702, 2176, 2177, 697, 1766, 692, 1766, - /* 2540 */ 2173, 1766, 2141, 1766, 698, 1766, 1766, 1766, 1766, 1766, - /* 2550 */ 699, 2191, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, - /* 2560 */ 1766, 1766, 1766, 2141, 1766, 698, 1766, 1766, 1766, 1766, - /* 2570 */ 1766, 1766, 1766, 1766, 1766, 1766, 2172, 1766, 2191, 2208, - /* 2580 */ 1766, 1766, 360, 2174, 702, 2176, 2177, 697, 1766, 692, - /* 2590 */ 2141, 1766, 698, 1766, 1766, 1766, 1766, 2172, 1766, 1766, - /* 2600 */ 2208, 2173, 1766, 356, 2174, 702, 2176, 2177, 697, 1766, - /* 2610 */ 692, 699, 1766, 1766, 1766, 1766, 1766, 1766, 2173, 1766, - /* 2620 */ 1766, 1766, 1766, 1766, 2172, 1766, 1766, 2208, 699, 1766, - /* 2630 */ 361, 2174, 702, 2176, 2177, 697, 1766, 692, 1766, 2191, - /* 2640 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, - /* 2650 */ 1766, 2141, 1766, 698, 1766, 1766, 2191, 1766, 1766, 1766, - /* 2660 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 2141, 1766, - /* 2670 */ 698, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, - /* 2680 */ 1766, 1766, 1766, 1766, 1766, 700, 1766, 1766, 2208, 1766, - /* 2690 */ 1766, 335, 2174, 702, 2176, 2177, 697, 1766, 692, 1766, - /* 2700 */ 1766, 1766, 2172, 1766, 1766, 2208, 1766, 1766, 334, 2174, - /* 2710 */ 702, 2176, 2177, 697, 1766, 692, + /* 1110 */ 740, 739, 738, 177, 176, 168, 682, 1946, 1888, 686, + /* 1120 */ 324, 2235, 734, 408, 407, 1990, 189, 2271, 2272, 2175, + /* 1130 */ 140, 2276, 371, 1555, 321, 73, 665, 2143, 72, 699, + /* 1140 */ 148, 2328, 135, 1542, 1622, 1540, 1548, 41, 40, 346, + /* 1150 */ 1211, 47, 45, 44, 43, 42, 1565, 1778, 682, 1946, + /* 1160 */ 216, 508, 506, 503, 41, 40, 1782, 2193, 47, 45, + /* 1170 */ 44, 43, 42, 1617, 2303, 1545, 1546, 2175, 298, 2143, + /* 1180 */ 1777, 698, 682, 1946, 41, 40, 1547, 699, 47, 45, + /* 1190 */ 44, 43, 42, 748, 682, 1946, 1908, 682, 1946, 633, + /* 1200 */ 62, 372, 679, 370, 369, 1598, 568, 2143, 1776, 469, + /* 1210 */ 1688, 688, 1569, 2174, 680, 2193, 2210, 304, 468, 110, + /* 1220 */ 2176, 702, 2178, 2179, 697, 1566, 692, 2143, 570, 698, + /* 1230 */ 2143, 2355, 569, 2263, 1999, 2175, 428, 395, 2259, 109, + /* 1240 */ 279, 403, 1775, 1731, 602, 699, 601, 2282, 273, 1997, + /* 1250 */ 598, 1685, 2153, 652, 1774, 682, 1946, 689, 2143, 645, + /* 1260 */ 1708, 2174, 2335, 596, 2210, 594, 2162, 333, 2176, 702, + /* 1270 */ 2178, 2179, 697, 2193, 692, 406, 2157, 644, 188, 81, + /* 1280 */ 80, 448, 2336, 646, 200, 2143, 2175, 698, 759, 757, + /* 1290 */ 249, 1773, 2143, 650, 248, 74, 699, 440, 438, 2283, + /* 1300 */ 1685, 1556, 619, 1551, 2143, 1772, 1208, 1209, 347, 572, + /* 1310 */ 642, 429, 155, 2159, 427, 423, 419, 416, 441, 2174, + /* 1320 */ 573, 137, 2210, 692, 2193, 110, 2176, 702, 2178, 2179, + /* 1330 */ 697, 1303, 692, 1559, 1561, 1886, 2143, 2355, 698, 2263, + /* 1340 */ 1924, 2143, 1301, 395, 2259, 82, 690, 1615, 1616, 1618, + /* 1350 */ 1619, 1620, 1621, 2175, 90, 2143, 191, 353, 239, 260, + /* 1360 */ 378, 237, 600, 699, 261, 1832, 1550, 155, 155, 648, + /* 1370 */ 2174, 241, 56, 2210, 240, 1598, 110, 2176, 702, 2178, + /* 1380 */ 2179, 697, 243, 692, 150, 242, 2175, 585, 2238, 245, + /* 1390 */ 2263, 2193, 244, 1823, 395, 2259, 699, 1821, 50, 741, + /* 1400 */ 50, 1262, 91, 2143, 266, 698, 1764, 1765, 155, 14, + /* 1410 */ 13, 1643, 50, 2175, 291, 587, 1549, 742, 1885, 590, + /* 1420 */ 106, 1281, 71, 699, 2193, 2194, 55, 1507, 1510, 153, + /* 1430 */ 103, 155, 412, 64, 1815, 1806, 2143, 2174, 698, 1279, + /* 1440 */ 2210, 1263, 50, 110, 2176, 702, 2178, 2179, 697, 2175, + /* 1450 */ 692, 2193, 1811, 2053, 50, 2236, 706, 2263, 1717, 699, + /* 1460 */ 1716, 395, 2259, 2143, 268, 698, 1987, 2293, 664, 153, + /* 1470 */ 2174, 155, 1465, 2210, 296, 2175, 110, 2176, 702, 2178, + /* 1480 */ 2179, 697, 674, 692, 783, 699, 658, 2193, 685, 300, + /* 1490 */ 2263, 1346, 653, 1659, 395, 2259, 136, 2174, 281, 2143, + /* 1500 */ 2210, 698, 1608, 111, 2176, 702, 2178, 2179, 697, 278, + /* 1510 */ 692, 1, 415, 2193, 316, 153, 1374, 2263, 5, 420, + /* 1520 */ 1572, 687, 2259, 1553, 362, 2143, 437, 698, 195, 1378, + /* 1530 */ 436, 1385, 196, 700, 439, 198, 2210, 1488, 311, 111, + /* 1540 */ 2176, 702, 2178, 2179, 697, 206, 692, 454, 1569, 2054, + /* 1550 */ 458, 491, 463, 2263, 2175, 1564, 1383, 357, 2259, 2174, + /* 1560 */ 476, 2046, 2210, 483, 699, 172, 2176, 702, 2178, 2179, + /* 1570 */ 697, 499, 692, 1552, 490, 158, 501, 492, 502, 210, + /* 1580 */ 2175, 211, 504, 505, 213, 507, 509, 1570, 524, 4, + /* 1590 */ 699, 525, 2193, 533, 532, 1567, 535, 221, 536, 223, + /* 1600 */ 1571, 537, 1573, 538, 2143, 1999, 698, 226, 544, 540, + /* 1610 */ 2117, 228, 364, 85, 86, 565, 647, 2356, 2193, 112, + /* 1620 */ 1997, 232, 567, 382, 2114, 604, 1936, 352, 2113, 606, + /* 1630 */ 2143, 312, 698, 89, 610, 151, 253, 609, 2174, 611, + /* 1640 */ 236, 2210, 2175, 1932, 111, 2176, 702, 2178, 2179, 697, + /* 1650 */ 238, 692, 699, 1495, 160, 161, 1934, 1930, 2263, 2175, + /* 1660 */ 257, 162, 163, 2260, 2174, 617, 614, 2210, 634, 699, + /* 1670 */ 340, 2176, 702, 2178, 2179, 697, 255, 692, 2175, 624, + /* 1680 */ 2193, 2294, 615, 645, 2309, 383, 2335, 2304, 699, 672, + /* 1690 */ 630, 264, 2143, 2308, 698, 267, 384, 2193, 637, 8, + /* 1700 */ 643, 644, 188, 385, 625, 623, 2336, 646, 175, 2143, + /* 1710 */ 2285, 698, 622, 654, 277, 2358, 2193, 651, 1568, 1685, + /* 1720 */ 662, 390, 141, 286, 2279, 388, 2174, 1574, 2143, 2210, + /* 1730 */ 698, 276, 340, 2176, 702, 2178, 2179, 697, 272, 692, + /* 1740 */ 96, 313, 2059, 2174, 675, 670, 2210, 671, 2073, 172, + /* 1750 */ 2176, 702, 2178, 2179, 697, 274, 692, 275, 314, 2072, + /* 1760 */ 61, 676, 2174, 2071, 2175, 2210, 391, 98, 340, 2176, + /* 1770 */ 702, 2178, 2179, 697, 696, 692, 2334, 315, 280, 2244, + /* 1780 */ 1947, 2175, 102, 704, 1991, 100, 784, 1909, 318, 307, + /* 1790 */ 785, 699, 53, 787, 342, 322, 354, 355, 2135, 2134, + /* 1800 */ 2175, 2357, 2193, 320, 327, 2133, 78, 341, 331, 2130, + /* 1810 */ 699, 417, 418, 1532, 2143, 1533, 698, 422, 194, 2193, + /* 1820 */ 2128, 424, 425, 426, 398, 2127, 363, 2125, 430, 2124, + /* 1830 */ 432, 2143, 2123, 698, 2104, 434, 1523, 197, 2193, 2103, + /* 1840 */ 199, 1491, 79, 400, 1490, 2085, 2084, 2083, 2174, 446, + /* 1850 */ 2143, 2210, 698, 447, 339, 2176, 702, 2178, 2179, 697, + /* 1860 */ 2082, 692, 2081, 2229, 1442, 2174, 147, 608, 2210, 2037, + /* 1870 */ 2036, 340, 2176, 702, 2178, 2179, 697, 2034, 692, 2033, + /* 1880 */ 2032, 2035, 2031, 2030, 2174, 791, 2028, 2210, 2027, 2026, + /* 1890 */ 340, 2176, 702, 2178, 2179, 697, 204, 692, 2175, 310, + /* 1900 */ 464, 2025, 466, 2039, 2024, 2023, 2022, 2021, 699, 2020, + /* 1910 */ 2019, 2018, 2017, 2016, 2015, 180, 149, 2009, 2008, 2007, + /* 1920 */ 2038, 2006, 2005, 781, 777, 773, 769, 2014, 308, 2013, + /* 1930 */ 2012, 2175, 2011, 2010, 2004, 2003, 2193, 1444, 2002, 494, + /* 1940 */ 2001, 699, 2000, 350, 351, 1317, 1321, 1851, 2143, 212, + /* 1950 */ 698, 1850, 214, 1849, 215, 1847, 1844, 511, 1843, 1836, + /* 1960 */ 1313, 510, 1825, 1801, 2175, 227, 514, 512, 108, 2193, + /* 1970 */ 63, 301, 1800, 2102, 699, 516, 518, 2092, 520, 522, + /* 1980 */ 2080, 2143, 603, 698, 1210, 2210, 219, 182, 335, 2176, + /* 1990 */ 702, 2178, 2179, 697, 515, 692, 217, 519, 2079, 76, + /* 2000 */ 2175, 2057, 2193, 1925, 678, 547, 77, 225, 2163, 1846, + /* 2010 */ 699, 183, 530, 1842, 2143, 2174, 698, 545, 2210, 1255, + /* 2020 */ 1840, 325, 2176, 702, 2178, 2179, 697, 546, 692, 549, + /* 2030 */ 550, 551, 1838, 2175, 553, 554, 555, 1835, 2193, 288, + /* 2040 */ 558, 557, 1820, 699, 287, 1818, 559, 1819, 2174, 1817, + /* 2050 */ 2143, 2210, 698, 1390, 323, 2176, 702, 2178, 2179, 697, + /* 2060 */ 1797, 692, 2175, 1927, 254, 1389, 1926, 235, 1304, 1302, + /* 2070 */ 1300, 2193, 699, 1299, 1298, 1297, 1296, 1291, 1833, 375, + /* 2080 */ 756, 1293, 1824, 2143, 2174, 698, 1292, 2210, 758, 1290, + /* 2090 */ 326, 2176, 702, 2178, 2179, 697, 376, 692, 1822, 588, + /* 2100 */ 2193, 377, 591, 1796, 593, 1795, 595, 1794, 597, 113, + /* 2110 */ 1517, 1519, 2143, 29, 698, 1516, 1521, 2174, 2101, 67, + /* 2120 */ 2210, 58, 2175, 332, 2176, 702, 2178, 2179, 697, 1497, + /* 2130 */ 692, 1499, 699, 2091, 612, 2078, 2076, 2340, 17, 20, + /* 2140 */ 6, 2175, 65, 613, 7, 165, 2174, 381, 258, 2210, + /* 2150 */ 21, 699, 336, 2176, 702, 2178, 2179, 697, 618, 692, + /* 2160 */ 2193, 31, 271, 1734, 263, 626, 1501, 628, 22, 265, + /* 2170 */ 270, 33, 2143, 173, 698, 2164, 1715, 1707, 1749, 2193, + /* 2180 */ 269, 24, 1748, 386, 32, 1753, 92, 283, 1752, 2175, + /* 2190 */ 387, 2143, 1754, 698, 1755, 1682, 178, 1681, 60, 699, + /* 2200 */ 94, 95, 25, 289, 290, 292, 2174, 2077, 2175, 2210, + /* 2210 */ 2075, 2074, 328, 2176, 702, 2178, 2179, 697, 699, 692, + /* 2220 */ 2056, 1713, 297, 68, 2055, 2174, 97, 2193, 2210, 299, + /* 2230 */ 302, 337, 2176, 702, 2178, 2179, 697, 673, 692, 2143, + /* 2240 */ 59, 698, 1848, 103, 99, 26, 2193, 23, 18, 13, + /* 2250 */ 1557, 1644, 1634, 179, 1589, 11, 2175, 705, 2143, 1633, + /* 2260 */ 698, 1612, 2213, 691, 192, 1610, 699, 39, 401, 1609, + /* 2270 */ 16, 27, 28, 2174, 1581, 707, 2210, 2175, 1375, 329, + /* 2280 */ 2176, 702, 2178, 2179, 697, 703, 692, 699, 1372, 709, + /* 2290 */ 1371, 710, 2174, 712, 2193, 2210, 713, 715, 338, 2176, + /* 2300 */ 702, 2178, 2179, 697, 718, 692, 2143, 1368, 698, 701, + /* 2310 */ 721, 716, 584, 583, 582, 2193, 1362, 719, 722, 574, + /* 2320 */ 139, 578, 1360, 1366, 1365, 577, 104, 2143, 305, 698, + /* 2330 */ 576, 581, 374, 373, 1364, 1384, 575, 105, 1363, 75, + /* 2340 */ 2174, 1380, 1253, 2210, 736, 2175, 330, 2176, 702, 2178, + /* 2350 */ 2179, 697, 1285, 692, 1311, 699, 1284, 1283, 1282, 1280, + /* 2360 */ 1278, 2174, 1277, 2175, 2210, 1276, 1308, 343, 2176, 702, + /* 2370 */ 2178, 2179, 697, 699, 692, 306, 1274, 746, 1273, 1272, + /* 2380 */ 1271, 1270, 1269, 2193, 1268, 1306, 1265, 1264, 1261, 1260, + /* 2390 */ 1259, 1258, 1841, 2175, 766, 2143, 768, 698, 1839, 770, + /* 2400 */ 772, 2193, 1837, 699, 767, 774, 776, 1834, 778, 771, + /* 2410 */ 780, 775, 779, 2143, 1816, 698, 782, 1200, 1793, 309, + /* 2420 */ 786, 1768, 1543, 319, 2175, 789, 790, 1768, 1768, 2174, + /* 2430 */ 1768, 2193, 2210, 1768, 699, 344, 2176, 702, 2178, 2179, + /* 2440 */ 697, 2175, 692, 2143, 1768, 698, 1768, 2174, 1768, 1768, + /* 2450 */ 2210, 699, 1768, 2187, 2176, 702, 2178, 2179, 697, 1768, + /* 2460 */ 692, 1768, 2193, 1768, 1768, 1768, 1768, 1768, 1768, 1768, + /* 2470 */ 1768, 1768, 1768, 1768, 2143, 1768, 698, 2174, 1768, 2193, + /* 2480 */ 2210, 1768, 1768, 2186, 2176, 702, 2178, 2179, 697, 1768, + /* 2490 */ 692, 2143, 2175, 698, 1768, 1768, 1768, 1768, 1768, 1768, + /* 2500 */ 1768, 1768, 699, 1768, 1768, 1768, 1768, 1768, 2174, 1768, + /* 2510 */ 1768, 2210, 1768, 2175, 2185, 2176, 702, 2178, 2179, 697, + /* 2520 */ 1768, 692, 1768, 699, 1768, 2174, 1768, 1768, 2210, 1768, + /* 2530 */ 2193, 359, 2176, 702, 2178, 2179, 697, 1768, 692, 1768, + /* 2540 */ 2175, 1768, 2143, 1768, 698, 1768, 1768, 1768, 1768, 1768, + /* 2550 */ 699, 2193, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, + /* 2560 */ 1768, 1768, 1768, 2143, 1768, 698, 1768, 1768, 1768, 1768, + /* 2570 */ 1768, 1768, 1768, 1768, 1768, 1768, 2174, 1768, 2193, 2210, + /* 2580 */ 1768, 1768, 360, 2176, 702, 2178, 2179, 697, 1768, 692, + /* 2590 */ 2143, 1768, 698, 1768, 1768, 1768, 1768, 2174, 1768, 1768, + /* 2600 */ 2210, 2175, 1768, 356, 2176, 702, 2178, 2179, 697, 1768, + /* 2610 */ 692, 699, 1768, 1768, 1768, 1768, 1768, 1768, 2175, 1768, + /* 2620 */ 1768, 1768, 1768, 1768, 2174, 1768, 1768, 2210, 699, 1768, + /* 2630 */ 361, 2176, 702, 2178, 2179, 697, 1768, 692, 1768, 2193, + /* 2640 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, + /* 2650 */ 1768, 2143, 1768, 698, 1768, 1768, 2193, 1768, 1768, 1768, + /* 2660 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 2143, 1768, + /* 2670 */ 698, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, + /* 2680 */ 1768, 1768, 1768, 1768, 1768, 700, 1768, 1768, 2210, 1768, + /* 2690 */ 1768, 335, 2176, 702, 2178, 2179, 697, 1768, 692, 1768, + /* 2700 */ 1768, 1768, 2174, 1768, 1768, 2210, 1768, 1768, 334, 2176, + /* 2710 */ 702, 2178, 2179, 697, 1768, 692, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 340, 359, 456, 445, 446, 459, 339, 409, 341, 367, @@ -514,229 +514,229 @@ static const YYCODETYPE yy_lookahead[] = { /* 170 */ 170, 171, 68, 140, 141, 390, 176, 177, 102, 75, /* 180 */ 0, 105, 349, 350, 169, 0, 172, 401, 75, 403, /* 190 */ 190, 21, 192, 20, 24, 25, 26, 27, 28, 29, - /* 200 */ 30, 31, 32, 172, 100, 349, 21, 103, 340, 24, + /* 200 */ 30, 31, 32, 172, 100, 49, 21, 103, 340, 24, /* 210 */ 25, 26, 27, 28, 29, 30, 31, 32, 350, 401, /* 220 */ 352, 403, 222, 223, 0, 225, 226, 227, 228, 229, /* 230 */ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, /* 240 */ 240, 241, 242, 20, 140, 141, 378, 67, 24, 25, - /* 250 */ 26, 27, 28, 29, 30, 31, 32, 401, 390, 403, + /* 250 */ 26, 27, 28, 29, 30, 31, 32, 0, 390, 22, /* 260 */ 392, 246, 247, 248, 249, 250, 251, 252, 253, 254, - /* 270 */ 255, 256, 67, 340, 170, 171, 103, 349, 350, 4, - /* 280 */ 176, 177, 39, 350, 451, 452, 453, 454, 172, 456, + /* 270 */ 255, 256, 35, 340, 170, 171, 103, 349, 350, 4, + /* 280 */ 176, 177, 20, 350, 451, 452, 453, 454, 172, 456, /* 290 */ 457, 20, 424, 22, 190, 427, 192, 369, 430, 431, /* 300 */ 432, 433, 434, 435, 376, 437, 35, 14, 349, 350, /* 310 */ 442, 378, 444, 20, 8, 9, 448, 449, 12, 13, /* 320 */ 14, 15, 16, 390, 53, 392, 222, 223, 369, 225, /* 330 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, /* 340 */ 236, 237, 238, 239, 240, 241, 242, 243, 12, 13, - /* 350 */ 20, 340, 22, 12, 13, 3, 20, 424, 22, 429, - /* 360 */ 427, 350, 0, 430, 431, 432, 433, 434, 435, 33, - /* 370 */ 437, 35, 216, 378, 378, 442, 35, 444, 103, 350, - /* 380 */ 385, 448, 449, 53, 392, 455, 8, 9, 393, 378, - /* 390 */ 12, 13, 14, 15, 16, 399, 404, 100, 62, 407, - /* 400 */ 408, 390, 469, 392, 68, 344, 44, 378, 347, 348, + /* 350 */ 44, 340, 440, 12, 13, 443, 20, 424, 22, 387, + /* 360 */ 427, 350, 390, 430, 431, 432, 433, 434, 435, 33, + /* 370 */ 437, 35, 185, 2, 47, 442, 35, 444, 103, 8, + /* 380 */ 9, 448, 449, 12, 13, 14, 15, 16, 103, 378, + /* 390 */ 133, 134, 135, 136, 137, 138, 139, 100, 62, 212, + /* 400 */ 213, 390, 469, 392, 68, 344, 20, 349, 347, 348, /* 410 */ 104, 75, 115, 116, 117, 118, 119, 120, 121, 122, - /* 420 */ 123, 124, 44, 126, 127, 128, 129, 130, 131, 132, - /* 430 */ 20, 14, 259, 20, 356, 424, 100, 20, 427, 103, - /* 440 */ 356, 430, 431, 432, 433, 434, 435, 436, 437, 438, - /* 450 */ 439, 373, 12, 13, 14, 358, 175, 2, 33, 381, - /* 460 */ 20, 432, 22, 8, 9, 381, 20, 12, 13, 14, - /* 470 */ 15, 16, 375, 33, 49, 35, 140, 141, 20, 349, - /* 480 */ 350, 384, 57, 58, 59, 60, 350, 62, 62, 370, - /* 490 */ 133, 134, 135, 136, 137, 138, 139, 378, 340, 369, - /* 500 */ 4, 258, 62, 20, 222, 386, 170, 171, 350, 243, - /* 510 */ 352, 245, 176, 177, 350, 75, 8, 9, 349, 350, - /* 520 */ 12, 13, 14, 15, 16, 20, 190, 102, 192, 103, - /* 530 */ 105, 105, 396, 192, 259, 84, 378, 20, 369, 43, - /* 540 */ 100, 45, 46, 103, 340, 264, 265, 266, 390, 103, + /* 420 */ 123, 124, 20, 126, 127, 128, 129, 130, 131, 132, + /* 430 */ 103, 14, 259, 20, 172, 424, 100, 20, 427, 103, + /* 440 */ 103, 430, 431, 432, 433, 434, 435, 436, 437, 438, + /* 450 */ 439, 340, 12, 13, 14, 358, 175, 2, 33, 401, + /* 460 */ 20, 403, 22, 8, 9, 378, 20, 12, 13, 14, + /* 470 */ 15, 16, 375, 33, 49, 35, 140, 141, 356, 349, + /* 480 */ 350, 384, 57, 58, 59, 60, 399, 62, 62, 103, + /* 490 */ 133, 134, 135, 136, 137, 138, 139, 67, 340, 369, + /* 500 */ 4, 390, 62, 381, 222, 103, 170, 171, 350, 429, + /* 510 */ 352, 358, 176, 177, 350, 75, 8, 9, 349, 350, + /* 520 */ 12, 13, 14, 15, 16, 67, 190, 102, 192, 103, + /* 530 */ 105, 105, 20, 192, 259, 455, 378, 384, 369, 43, + /* 540 */ 100, 45, 46, 103, 259, 264, 265, 266, 390, 20, /* 550 */ 392, 269, 270, 271, 272, 273, 274, 275, 222, 223, /* 560 */ 396, 225, 226, 227, 228, 229, 230, 231, 232, 233, - /* 570 */ 234, 235, 236, 237, 238, 239, 240, 241, 242, 103, - /* 580 */ 140, 141, 424, 170, 171, 427, 103, 170, 430, 431, - /* 590 */ 432, 433, 434, 435, 390, 437, 145, 146, 173, 174, - /* 600 */ 442, 49, 444, 178, 67, 180, 448, 449, 340, 57, - /* 610 */ 170, 171, 60, 61, 349, 350, 176, 177, 350, 168, - /* 620 */ 352, 349, 350, 198, 114, 366, 103, 103, 134, 135, - /* 630 */ 190, 279, 192, 139, 369, 70, 71, 72, 441, 380, - /* 640 */ 443, 369, 77, 78, 79, 0, 378, 340, 83, 390, - /* 650 */ 349, 350, 35, 88, 89, 90, 91, 0, 390, 94, + /* 570 */ 234, 235, 236, 237, 238, 239, 240, 241, 242, 378, + /* 580 */ 140, 141, 424, 170, 171, 427, 385, 170, 430, 431, + /* 590 */ 432, 433, 434, 435, 393, 437, 259, 392, 173, 174, + /* 600 */ 442, 114, 444, 178, 378, 180, 448, 449, 340, 404, + /* 610 */ 170, 171, 407, 408, 349, 350, 176, 177, 350, 393, + /* 620 */ 352, 349, 350, 198, 243, 366, 429, 103, 134, 135, + /* 630 */ 190, 75, 192, 139, 369, 70, 71, 72, 243, 380, + /* 640 */ 245, 369, 77, 78, 79, 259, 378, 340, 83, 390, + /* 650 */ 349, 350, 455, 88, 89, 90, 91, 169, 390, 94, /* 660 */ 392, 409, 222, 223, 20, 225, 226, 227, 228, 229, /* 670 */ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, - /* 680 */ 240, 241, 242, 12, 13, 259, 427, 349, 350, 172, - /* 690 */ 182, 20, 424, 22, 49, 427, 437, 390, 430, 431, + /* 680 */ 240, 241, 242, 12, 13, 259, 427, 349, 350, 350, + /* 690 */ 182, 20, 424, 22, 114, 427, 437, 390, 430, 431, /* 700 */ 432, 433, 434, 435, 33, 437, 35, 369, 456, 340, /* 710 */ 442, 459, 444, 189, 370, 191, 448, 449, 0, 350, - /* 720 */ 388, 4, 378, 391, 392, 429, 474, 475, 349, 350, - /* 730 */ 386, 479, 480, 62, 14, 259, 19, 391, 392, 340, - /* 740 */ 20, 185, 259, 349, 350, 221, 75, 378, 369, 350, - /* 750 */ 33, 455, 21, 452, 453, 454, 243, 456, 457, 390, - /* 760 */ 459, 392, 370, 369, 429, 34, 49, 36, 212, 213, - /* 770 */ 378, 100, 75, 56, 103, 474, 475, 378, 386, 62, - /* 780 */ 479, 480, 259, 259, 140, 141, 75, 12, 13, 390, - /* 790 */ 455, 392, 284, 424, 22, 20, 427, 22, 134, 430, - /* 800 */ 431, 432, 433, 434, 435, 169, 437, 35, 33, 192, - /* 810 */ 35, 140, 141, 444, 349, 350, 159, 448, 449, 102, - /* 820 */ 176, 177, 105, 424, 0, 168, 427, 114, 378, 430, - /* 830 */ 431, 432, 433, 434, 435, 385, 437, 62, 340, 340, - /* 840 */ 14, 170, 171, 393, 349, 350, 20, 176, 177, 350, - /* 850 */ 75, 133, 134, 135, 136, 137, 138, 139, 22, 378, - /* 860 */ 441, 190, 443, 192, 369, 44, 385, 203, 204, 470, - /* 870 */ 471, 35, 100, 2, 393, 100, 379, 378, 103, 8, - /* 880 */ 9, 379, 246, 12, 13, 14, 15, 16, 390, 390, - /* 890 */ 170, 392, 256, 222, 223, 379, 225, 226, 227, 228, + /* 720 */ 388, 4, 378, 391, 392, 371, 474, 475, 374, 49, + /* 730 */ 386, 479, 480, 62, 246, 396, 19, 57, 22, 340, + /* 740 */ 60, 61, 349, 350, 256, 221, 75, 378, 370, 350, + /* 750 */ 33, 35, 75, 452, 453, 454, 378, 456, 457, 390, + /* 760 */ 459, 392, 369, 378, 386, 356, 49, 391, 392, 387, + /* 770 */ 385, 100, 390, 56, 103, 474, 475, 378, 393, 62, + /* 780 */ 479, 480, 373, 259, 140, 141, 379, 12, 13, 390, + /* 790 */ 381, 392, 284, 424, 379, 20, 427, 22, 84, 430, + /* 800 */ 431, 432, 433, 434, 435, 20, 437, 22, 33, 429, + /* 810 */ 35, 140, 141, 444, 349, 350, 100, 448, 449, 102, + /* 820 */ 176, 177, 105, 424, 349, 350, 427, 354, 355, 430, + /* 830 */ 431, 432, 433, 434, 435, 455, 437, 62, 53, 340, + /* 840 */ 340, 170, 171, 350, 369, 1, 2, 176, 177, 350, + /* 850 */ 75, 133, 134, 135, 136, 137, 138, 139, 378, 145, + /* 860 */ 146, 190, 441, 192, 443, 385, 14, 15, 16, 470, + /* 870 */ 471, 378, 133, 393, 340, 100, 137, 378, 103, 8, + /* 880 */ 9, 379, 168, 12, 13, 14, 15, 16, 340, 390, + /* 890 */ 390, 392, 409, 222, 223, 379, 225, 226, 227, 228, /* 900 */ 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, - /* 910 */ 239, 240, 241, 242, 340, 140, 141, 452, 453, 454, - /* 920 */ 337, 456, 457, 424, 340, 104, 427, 340, 340, 430, - /* 930 */ 431, 432, 433, 434, 435, 18, 437, 20, 14, 15, - /* 940 */ 16, 378, 414, 340, 27, 170, 171, 30, 385, 340, - /* 950 */ 33, 176, 177, 350, 378, 352, 393, 133, 134, 135, - /* 960 */ 136, 137, 138, 139, 390, 190, 49, 192, 51, 393, - /* 970 */ 471, 349, 350, 56, 390, 22, 22, 390, 390, 8, - /* 980 */ 9, 378, 340, 12, 13, 14, 15, 16, 35, 35, - /* 990 */ 20, 369, 409, 390, 340, 392, 413, 222, 223, 390, + /* 910 */ 239, 240, 241, 242, 409, 140, 141, 452, 453, 454, + /* 920 */ 337, 456, 457, 424, 390, 432, 427, 340, 340, 430, + /* 930 */ 431, 432, 433, 434, 435, 18, 437, 20, 390, 456, + /* 940 */ 349, 350, 459, 340, 27, 170, 171, 30, 104, 340, + /* 950 */ 33, 176, 177, 350, 4, 352, 134, 474, 475, 414, + /* 960 */ 369, 456, 479, 480, 459, 190, 49, 192, 51, 379, + /* 970 */ 471, 349, 350, 56, 370, 22, 22, 390, 390, 474, + /* 980 */ 475, 378, 378, 350, 479, 480, 54, 55, 35, 35, + /* 990 */ 386, 369, 409, 390, 340, 392, 413, 222, 223, 390, /* 1000 */ 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, /* 1010 */ 235, 236, 237, 238, 239, 240, 241, 242, 0, 102, - /* 1020 */ 20, 349, 350, 349, 350, 363, 364, 424, 75, 379, - /* 1030 */ 427, 114, 390, 430, 431, 432, 433, 434, 435, 456, - /* 1040 */ 437, 369, 459, 340, 390, 442, 133, 444, 54, 55, - /* 1050 */ 137, 448, 449, 100, 100, 354, 355, 474, 475, 354, + /* 1020 */ 0, 349, 350, 349, 350, 203, 204, 424, 75, 396, + /* 1030 */ 427, 114, 39, 430, 431, 432, 433, 434, 435, 456, + /* 1040 */ 437, 369, 459, 172, 390, 442, 42, 444, 44, 14, + /* 1050 */ 0, 448, 449, 100, 100, 20, 44, 474, 475, 354, /* 1060 */ 355, 144, 479, 480, 147, 148, 149, 150, 151, 152, /* 1070 */ 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, - /* 1080 */ 163, 44, 165, 166, 167, 363, 364, 340, 70, 71, - /* 1090 */ 72, 73, 74, 390, 76, 77, 78, 79, 80, 81, + /* 1080 */ 163, 379, 165, 166, 167, 363, 364, 340, 70, 71, + /* 1090 */ 72, 73, 74, 3, 76, 77, 78, 79, 80, 81, /* 1100 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - /* 1110 */ 92, 93, 94, 95, 96, 18, 349, 350, 409, 340, - /* 1120 */ 23, 340, 340, 12, 13, 379, 452, 453, 454, 340, + /* 1110 */ 92, 93, 94, 95, 96, 18, 349, 350, 367, 441, + /* 1120 */ 23, 443, 387, 12, 13, 390, 452, 453, 454, 340, /* 1130 */ 456, 457, 37, 22, 37, 38, 369, 390, 41, 350, - /* 1140 */ 379, 352, 172, 190, 33, 192, 35, 8, 9, 52, - /* 1150 */ 4, 12, 13, 14, 15, 16, 350, 340, 349, 350, - /* 1160 */ 63, 64, 65, 66, 42, 456, 44, 378, 459, 390, - /* 1170 */ 170, 390, 390, 62, 44, 222, 223, 340, 369, 390, - /* 1180 */ 340, 392, 62, 474, 475, 366, 75, 350, 479, 480, - /* 1190 */ 8, 9, 1, 2, 12, 13, 14, 15, 16, 380, - /* 1200 */ 103, 106, 396, 108, 109, 114, 111, 390, 358, 390, - /* 1210 */ 68, 100, 409, 424, 440, 378, 427, 443, 367, 430, - /* 1220 */ 431, 432, 433, 434, 435, 105, 437, 390, 133, 392, - /* 1230 */ 390, 442, 137, 444, 384, 340, 13, 448, 449, 142, - /* 1240 */ 42, 387, 44, 104, 390, 350, 427, 352, 387, 387, - /* 1250 */ 371, 390, 390, 374, 365, 164, 437, 368, 35, 456, - /* 1260 */ 107, 424, 459, 110, 427, 257, 258, 430, 431, 432, - /* 1270 */ 433, 434, 435, 378, 437, 45, 46, 474, 475, 182, - /* 1280 */ 183, 184, 479, 480, 187, 390, 340, 392, 107, 341, - /* 1290 */ 13, 110, 48, 349, 350, 104, 350, 200, 201, 107, - /* 1300 */ 400, 190, 110, 192, 107, 140, 141, 110, 211, 0, - /* 1310 */ 473, 214, 35, 369, 217, 218, 219, 220, 221, 424, - /* 1320 */ 283, 206, 427, 208, 378, 430, 431, 432, 433, 434, - /* 1330 */ 435, 22, 437, 222, 223, 0, 390, 442, 392, 444, - /* 1340 */ 0, 44, 456, 448, 449, 459, 235, 236, 237, 238, - /* 1350 */ 239, 240, 241, 340, 172, 135, 259, 22, 44, 139, - /* 1360 */ 474, 475, 22, 350, 222, 479, 480, 483, 44, 44, - /* 1370 */ 424, 44, 44, 427, 44, 44, 430, 431, 432, 433, - /* 1380 */ 434, 435, 472, 437, 349, 350, 340, 353, 442, 103, - /* 1390 */ 444, 378, 349, 350, 448, 449, 350, 349, 350, 113, - /* 1400 */ 44, 104, 47, 390, 369, 392, 260, 44, 44, 1, - /* 1410 */ 2, 281, 369, 340, 44, 466, 172, 369, 104, 199, - /* 1420 */ 35, 0, 202, 350, 378, 205, 44, 207, 104, 104, - /* 1430 */ 13, 104, 104, 44, 104, 104, 390, 424, 392, 378, - /* 1440 */ 427, 366, 44, 430, 431, 432, 433, 434, 435, 340, - /* 1450 */ 437, 378, 35, 366, 44, 442, 44, 444, 103, 350, - /* 1460 */ 104, 448, 449, 390, 44, 392, 353, 104, 104, 44, - /* 1470 */ 424, 50, 35, 427, 104, 340, 430, 431, 432, 433, - /* 1480 */ 434, 435, 400, 437, 348, 350, 104, 378, 442, 13, - /* 1490 */ 444, 350, 389, 104, 448, 449, 400, 424, 458, 390, - /* 1500 */ 427, 392, 104, 430, 431, 432, 433, 434, 435, 476, - /* 1510 */ 437, 35, 75, 378, 104, 450, 104, 444, 460, 261, - /* 1520 */ 426, 448, 449, 49, 104, 390, 425, 392, 20, 104, - /* 1530 */ 418, 205, 358, 424, 418, 358, 427, 423, 188, 430, - /* 1540 */ 431, 432, 433, 434, 435, 411, 437, 42, 397, 20, - /* 1550 */ 400, 397, 169, 444, 340, 395, 20, 448, 449, 424, + /* 1140 */ 42, 352, 44, 190, 33, 192, 35, 8, 9, 52, + /* 1150 */ 14, 12, 13, 14, 15, 16, 20, 340, 349, 350, + /* 1160 */ 63, 64, 65, 66, 8, 9, 341, 378, 12, 13, + /* 1170 */ 14, 15, 16, 62, 400, 222, 223, 340, 369, 390, + /* 1180 */ 340, 392, 349, 350, 8, 9, 75, 350, 12, 13, + /* 1190 */ 14, 15, 16, 365, 349, 350, 368, 349, 350, 472, + /* 1200 */ 103, 106, 369, 108, 109, 170, 111, 390, 340, 159, + /* 1210 */ 260, 100, 20, 424, 369, 378, 427, 369, 168, 430, + /* 1220 */ 431, 432, 433, 434, 435, 20, 437, 390, 133, 392, + /* 1230 */ 390, 442, 137, 444, 378, 340, 216, 448, 449, 142, + /* 1240 */ 483, 385, 340, 104, 206, 350, 208, 352, 466, 393, + /* 1250 */ 21, 258, 366, 44, 340, 349, 350, 68, 390, 456, + /* 1260 */ 104, 424, 459, 34, 427, 36, 380, 430, 431, 432, + /* 1270 */ 433, 434, 435, 378, 437, 369, 390, 474, 475, 182, + /* 1280 */ 183, 184, 479, 480, 187, 390, 340, 392, 363, 364, + /* 1290 */ 135, 340, 390, 281, 139, 114, 350, 200, 201, 257, + /* 1300 */ 258, 190, 48, 192, 390, 340, 45, 46, 211, 13, + /* 1310 */ 473, 214, 44, 427, 217, 218, 219, 220, 221, 424, + /* 1320 */ 13, 353, 427, 437, 378, 430, 431, 432, 433, 434, + /* 1330 */ 435, 35, 437, 222, 223, 366, 390, 442, 392, 444, + /* 1340 */ 0, 390, 35, 448, 449, 164, 235, 236, 237, 238, + /* 1350 */ 239, 240, 241, 340, 199, 390, 259, 202, 107, 62, + /* 1360 */ 205, 110, 207, 350, 172, 0, 35, 44, 44, 279, + /* 1370 */ 424, 107, 104, 427, 110, 170, 430, 431, 432, 433, + /* 1380 */ 434, 435, 107, 437, 44, 110, 340, 22, 442, 107, + /* 1390 */ 444, 378, 110, 0, 448, 449, 350, 0, 44, 13, + /* 1400 */ 44, 35, 105, 390, 44, 392, 140, 141, 44, 1, + /* 1410 */ 2, 222, 44, 340, 44, 22, 35, 13, 366, 22, + /* 1420 */ 103, 35, 44, 350, 378, 378, 172, 104, 104, 44, + /* 1430 */ 113, 44, 353, 44, 0, 348, 390, 424, 392, 35, + /* 1440 */ 427, 75, 44, 430, 431, 432, 433, 434, 435, 340, + /* 1450 */ 437, 378, 350, 400, 44, 442, 44, 444, 104, 350, + /* 1460 */ 104, 448, 449, 390, 104, 392, 389, 400, 104, 44, + /* 1470 */ 424, 44, 104, 427, 104, 340, 430, 431, 432, 433, + /* 1480 */ 434, 435, 104, 437, 50, 350, 458, 378, 442, 104, + /* 1490 */ 444, 104, 283, 104, 448, 449, 44, 424, 476, 390, + /* 1500 */ 427, 392, 104, 430, 431, 432, 433, 434, 435, 450, + /* 1510 */ 437, 460, 426, 378, 104, 44, 104, 444, 261, 49, + /* 1520 */ 20, 448, 449, 192, 425, 390, 418, 392, 423, 104, + /* 1530 */ 205, 104, 358, 424, 418, 358, 427, 188, 411, 430, + /* 1540 */ 431, 432, 433, 434, 435, 42, 437, 397, 20, 400, + /* 1550 */ 397, 169, 395, 444, 340, 20, 104, 448, 449, 424, /* 1560 */ 349, 349, 427, 397, 350, 430, 431, 432, 433, 434, - /* 1570 */ 435, 395, 437, 395, 101, 362, 99, 192, 361, 98, - /* 1580 */ 340, 349, 360, 349, 349, 349, 20, 342, 48, 346, - /* 1590 */ 350, 342, 378, 346, 20, 358, 392, 418, 20, 358, - /* 1600 */ 20, 351, 342, 410, 390, 378, 392, 351, 349, 358, - /* 1610 */ 358, 358, 385, 358, 390, 358, 481, 482, 378, 349, - /* 1620 */ 393, 390, 342, 383, 378, 209, 378, 422, 390, 103, - /* 1630 */ 390, 196, 392, 195, 418, 420, 417, 356, 424, 378, + /* 1570 */ 435, 99, 437, 192, 395, 104, 101, 395, 362, 361, + /* 1580 */ 340, 349, 98, 360, 349, 349, 349, 20, 342, 48, + /* 1590 */ 350, 346, 378, 346, 342, 20, 418, 358, 392, 358, + /* 1600 */ 20, 351, 20, 410, 390, 378, 392, 358, 349, 351, + /* 1610 */ 390, 358, 385, 358, 358, 342, 481, 482, 378, 349, + /* 1620 */ 393, 358, 378, 383, 390, 209, 378, 342, 390, 422, + /* 1630 */ 390, 418, 392, 103, 196, 420, 356, 195, 424, 417, /* 1640 */ 378, 427, 340, 378, 430, 431, 432, 433, 434, 435, - /* 1650 */ 378, 437, 350, 378, 378, 416, 378, 378, 444, 340, - /* 1660 */ 378, 392, 356, 449, 424, 194, 349, 427, 400, 350, - /* 1670 */ 430, 431, 432, 433, 434, 435, 267, 437, 340, 415, - /* 1680 */ 378, 390, 268, 456, 400, 383, 459, 276, 350, 390, - /* 1690 */ 465, 465, 390, 405, 392, 405, 468, 378, 390, 181, - /* 1700 */ 390, 474, 475, 285, 278, 277, 479, 480, 262, 390, - /* 1710 */ 258, 392, 282, 426, 280, 350, 378, 484, 20, 465, - /* 1720 */ 478, 383, 429, 349, 351, 356, 424, 464, 390, 427, - /* 1730 */ 392, 356, 430, 431, 432, 433, 434, 435, 467, 437, - /* 1740 */ 20, 405, 403, 424, 477, 462, 427, 390, 463, 430, - /* 1750 */ 431, 432, 433, 434, 435, 174, 437, 390, 390, 405, - /* 1760 */ 402, 390, 424, 356, 340, 427, 390, 390, 430, 431, - /* 1770 */ 432, 433, 434, 435, 350, 437, 374, 350, 356, 103, - /* 1780 */ 447, 340, 103, 382, 390, 349, 36, 368, 356, 338, - /* 1790 */ 343, 350, 342, 412, 406, 372, 406, 419, 0, 357, - /* 1800 */ 340, 482, 378, 372, 0, 372, 0, 42, 0, 35, - /* 1810 */ 350, 215, 35, 35, 390, 35, 392, 215, 0, 378, - /* 1820 */ 35, 35, 215, 0, 383, 215, 0, 35, 0, 22, - /* 1830 */ 0, 390, 35, 392, 0, 210, 198, 0, 378, 198, - /* 1840 */ 192, 199, 190, 383, 0, 0, 0, 186, 424, 185, - /* 1850 */ 390, 427, 392, 0, 430, 431, 432, 433, 434, 435, - /* 1860 */ 0, 437, 47, 439, 0, 424, 42, 1, 427, 0, + /* 1650 */ 378, 437, 350, 194, 378, 378, 378, 378, 444, 340, + /* 1660 */ 356, 378, 378, 449, 424, 349, 392, 427, 268, 350, + /* 1670 */ 430, 431, 432, 433, 434, 435, 416, 437, 340, 390, + /* 1680 */ 378, 400, 415, 456, 465, 383, 459, 400, 350, 267, + /* 1690 */ 390, 405, 390, 465, 392, 405, 390, 378, 390, 276, + /* 1700 */ 181, 474, 475, 285, 278, 277, 479, 480, 465, 390, + /* 1710 */ 468, 392, 262, 282, 426, 484, 378, 280, 20, 258, + /* 1720 */ 349, 383, 350, 356, 429, 351, 424, 20, 390, 427, + /* 1730 */ 392, 462, 430, 431, 432, 433, 434, 435, 467, 437, + /* 1740 */ 356, 405, 403, 424, 174, 390, 427, 390, 390, 430, + /* 1750 */ 431, 432, 433, 434, 435, 464, 437, 463, 405, 390, + /* 1760 */ 103, 402, 424, 390, 340, 427, 390, 356, 430, 431, + /* 1770 */ 432, 433, 434, 435, 350, 437, 478, 374, 477, 447, + /* 1780 */ 350, 340, 103, 382, 390, 356, 36, 368, 349, 356, + /* 1790 */ 343, 350, 412, 342, 419, 338, 406, 406, 0, 0, + /* 1800 */ 340, 482, 378, 357, 372, 0, 42, 372, 372, 0, + /* 1810 */ 350, 35, 215, 35, 390, 35, 392, 215, 35, 378, + /* 1820 */ 0, 35, 35, 215, 383, 0, 215, 0, 35, 0, + /* 1830 */ 22, 390, 0, 392, 0, 35, 210, 198, 378, 0, + /* 1840 */ 198, 192, 199, 383, 190, 0, 0, 0, 424, 186, + /* 1850 */ 390, 427, 392, 185, 430, 431, 432, 433, 434, 435, + /* 1860 */ 0, 437, 0, 439, 47, 424, 42, 1, 427, 0, /* 1870 */ 0, 430, 431, 432, 433, 434, 435, 0, 437, 0, - /* 1880 */ 0, 0, 0, 0, 424, 19, 0, 427, 0, 159, - /* 1890 */ 430, 431, 432, 433, 434, 435, 35, 437, 340, 33, - /* 1900 */ 0, 159, 0, 0, 0, 0, 0, 0, 350, 0, + /* 1880 */ 0, 0, 0, 0, 424, 19, 0, 427, 0, 0, + /* 1890 */ 430, 431, 432, 433, 434, 435, 159, 437, 340, 33, + /* 1900 */ 35, 0, 159, 0, 0, 0, 0, 0, 350, 0, /* 1910 */ 0, 0, 0, 0, 0, 49, 42, 0, 0, 0, /* 1920 */ 0, 0, 0, 57, 58, 59, 60, 0, 62, 0, - /* 1930 */ 0, 340, 0, 0, 0, 0, 378, 22, 143, 0, - /* 1940 */ 0, 350, 48, 48, 0, 22, 22, 62, 390, 0, - /* 1950 */ 392, 62, 0, 62, 0, 0, 0, 49, 0, 0, + /* 1930 */ 0, 340, 0, 0, 0, 0, 378, 22, 0, 143, + /* 1940 */ 0, 350, 0, 48, 48, 22, 22, 0, 390, 62, + /* 1950 */ 392, 0, 62, 0, 62, 0, 0, 49, 0, 0, /* 1960 */ 35, 35, 0, 0, 340, 181, 35, 39, 102, 378, - /* 1970 */ 0, 105, 0, 0, 350, 39, 35, 35, 0, 14, - /* 1980 */ 0, 390, 424, 392, 0, 427, 39, 49, 430, 431, - /* 1990 */ 432, 433, 434, 435, 49, 437, 0, 44, 47, 42, - /* 2000 */ 340, 0, 378, 39, 138, 39, 39, 47, 47, 40, - /* 2010 */ 350, 35, 69, 39, 390, 424, 392, 49, 427, 0, - /* 2020 */ 35, 430, 431, 432, 433, 434, 435, 39, 437, 0, - /* 2030 */ 49, 49, 35, 340, 39, 0, 49, 35, 378, 173, - /* 2040 */ 0, 0, 39, 350, 178, 0, 0, 0, 424, 0, + /* 1970 */ 112, 105, 0, 0, 350, 39, 35, 0, 39, 35, + /* 1980 */ 0, 390, 424, 392, 14, 427, 40, 44, 430, 431, + /* 1990 */ 432, 433, 434, 435, 49, 437, 42, 49, 0, 39, + /* 2000 */ 340, 0, 378, 0, 138, 39, 39, 39, 47, 0, + /* 2010 */ 350, 47, 47, 0, 390, 424, 392, 35, 427, 69, + /* 2020 */ 0, 430, 431, 432, 433, 434, 435, 49, 437, 35, + /* 2030 */ 49, 39, 0, 340, 35, 49, 39, 0, 378, 173, + /* 2040 */ 49, 35, 0, 350, 178, 0, 39, 0, 424, 0, /* 2050 */ 390, 427, 392, 35, 430, 431, 432, 433, 434, 435, - /* 2060 */ 112, 437, 340, 0, 198, 22, 35, 35, 35, 35, - /* 2070 */ 44, 378, 350, 35, 35, 0, 22, 35, 22, 35, - /* 2080 */ 44, 110, 0, 390, 424, 392, 35, 427, 22, 0, - /* 2090 */ 430, 431, 432, 433, 434, 435, 35, 437, 22, 51, - /* 2100 */ 378, 22, 35, 0, 35, 0, 35, 0, 20, 35, - /* 2110 */ 104, 103, 390, 35, 392, 35, 0, 424, 172, 103, - /* 2120 */ 427, 35, 340, 430, 431, 432, 433, 434, 435, 0, - /* 2130 */ 437, 22, 350, 22, 0, 0, 3, 44, 263, 48, - /* 2140 */ 103, 340, 48, 44, 44, 47, 424, 197, 44, 427, - /* 2150 */ 44, 350, 430, 431, 432, 433, 434, 435, 172, 437, - /* 2160 */ 378, 104, 172, 174, 104, 103, 101, 99, 193, 47, - /* 2170 */ 179, 3, 390, 103, 392, 44, 104, 104, 35, 378, - /* 2180 */ 103, 35, 35, 35, 103, 35, 103, 47, 35, 340, - /* 2190 */ 0, 390, 47, 392, 104, 44, 104, 0, 0, 350, - /* 2200 */ 104, 104, 0, 39, 47, 103, 424, 0, 340, 427, - /* 2210 */ 39, 103, 430, 431, 432, 433, 434, 435, 350, 437, - /* 2220 */ 44, 104, 104, 103, 103, 424, 103, 378, 427, 103, - /* 2230 */ 175, 430, 431, 432, 433, 434, 435, 173, 437, 390, - /* 2240 */ 263, 392, 0, 113, 244, 101, 378, 2, 47, 22, - /* 2250 */ 257, 47, 222, 104, 22, 263, 340, 101, 390, 104, - /* 2260 */ 392, 103, 103, 103, 47, 104, 350, 104, 103, 103, - /* 2270 */ 103, 35, 35, 424, 224, 103, 427, 340, 35, 430, - /* 2280 */ 431, 432, 433, 434, 435, 104, 437, 350, 114, 104, - /* 2290 */ 103, 35, 424, 104, 378, 427, 103, 35, 430, 431, - /* 2300 */ 432, 433, 434, 435, 35, 437, 390, 104, 392, 103, - /* 2310 */ 35, 103, 70, 71, 72, 378, 104, 104, 103, 77, - /* 2320 */ 78, 79, 103, 44, 125, 83, 103, 390, 35, 392, - /* 2330 */ 88, 89, 90, 91, 125, 125, 94, 103, 22, 125, - /* 2340 */ 424, 69, 68, 427, 35, 340, 430, 431, 432, 433, - /* 2350 */ 434, 435, 35, 437, 35, 350, 35, 35, 35, 35, - /* 2360 */ 35, 424, 75, 340, 427, 97, 22, 430, 431, 432, - /* 2370 */ 433, 434, 435, 350, 437, 44, 35, 35, 35, 75, - /* 2380 */ 35, 35, 35, 378, 35, 35, 35, 35, 35, 22, - /* 2390 */ 35, 0, 35, 340, 0, 390, 35, 392, 39, 39, - /* 2400 */ 0, 378, 39, 350, 35, 49, 0, 49, 35, 39, - /* 2410 */ 0, 49, 35, 390, 49, 392, 35, 0, 22, 21, - /* 2420 */ 485, 22, 20, 22, 340, 21, 485, 485, 485, 424, + /* 2060 */ 0, 437, 340, 0, 198, 22, 0, 110, 35, 35, + /* 2070 */ 35, 378, 350, 35, 35, 35, 35, 22, 0, 22, + /* 2080 */ 44, 35, 0, 390, 424, 392, 35, 427, 44, 35, + /* 2090 */ 430, 431, 432, 433, 434, 435, 22, 437, 0, 51, + /* 2100 */ 378, 22, 35, 0, 35, 0, 35, 0, 22, 20, + /* 2110 */ 35, 35, 390, 103, 392, 35, 104, 424, 0, 103, + /* 2120 */ 427, 172, 340, 430, 431, 432, 433, 434, 435, 35, + /* 2130 */ 437, 22, 350, 0, 22, 0, 0, 3, 263, 44, + /* 2140 */ 48, 340, 3, 172, 48, 193, 424, 172, 174, 427, + /* 2150 */ 44, 350, 430, 431, 432, 433, 434, 435, 179, 437, + /* 2160 */ 378, 103, 47, 104, 103, 101, 197, 99, 44, 104, + /* 2170 */ 44, 44, 390, 103, 392, 47, 104, 104, 35, 378, + /* 2180 */ 103, 44, 35, 35, 103, 35, 103, 47, 35, 340, + /* 2190 */ 35, 390, 104, 392, 104, 104, 47, 104, 44, 350, + /* 2200 */ 103, 39, 103, 47, 104, 103, 424, 0, 340, 427, + /* 2210 */ 0, 0, 430, 431, 432, 433, 434, 435, 350, 437, + /* 2220 */ 0, 104, 103, 103, 0, 424, 39, 378, 427, 173, + /* 2230 */ 47, 430, 431, 432, 433, 434, 435, 175, 437, 390, + /* 2240 */ 257, 392, 0, 113, 103, 44, 378, 263, 263, 2, + /* 2250 */ 22, 222, 101, 47, 22, 244, 340, 35, 390, 101, + /* 2260 */ 392, 104, 103, 103, 47, 104, 350, 103, 35, 104, + /* 2270 */ 103, 103, 103, 424, 104, 103, 427, 340, 104, 430, + /* 2280 */ 431, 432, 433, 434, 435, 114, 437, 350, 104, 35, + /* 2290 */ 104, 103, 424, 35, 378, 427, 103, 35, 430, 431, + /* 2300 */ 432, 433, 434, 435, 35, 437, 390, 104, 392, 224, + /* 2310 */ 35, 103, 70, 71, 72, 378, 104, 103, 103, 77, + /* 2320 */ 78, 79, 104, 125, 125, 83, 103, 390, 44, 392, + /* 2330 */ 88, 89, 90, 91, 125, 35, 94, 103, 125, 103, + /* 2340 */ 424, 22, 69, 427, 68, 340, 430, 431, 432, 433, + /* 2350 */ 434, 435, 35, 437, 75, 350, 35, 35, 35, 35, + /* 2360 */ 35, 424, 35, 340, 427, 35, 75, 430, 431, 432, + /* 2370 */ 433, 434, 435, 350, 437, 44, 35, 97, 35, 35, + /* 2380 */ 22, 35, 35, 378, 35, 35, 35, 35, 35, 35, + /* 2390 */ 22, 35, 0, 340, 35, 390, 39, 392, 0, 35, + /* 2400 */ 39, 378, 0, 350, 49, 35, 39, 0, 35, 49, + /* 2410 */ 39, 49, 49, 390, 0, 392, 35, 35, 0, 22, + /* 2420 */ 21, 485, 22, 22, 340, 21, 20, 485, 485, 424, /* 2430 */ 485, 378, 427, 485, 350, 430, 431, 432, 433, 434, /* 2440 */ 435, 340, 437, 390, 485, 392, 485, 424, 485, 485, /* 2450 */ 427, 350, 485, 430, 431, 432, 433, 434, 435, 485, @@ -770,7 +770,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 2730 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, /* 2740 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, /* 2750 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2760 */ 485, 485, 485, 337, 337, 337, 337, 337, 337, 337, + /* 2760 */ 485, 485, 485, 485, 337, 337, 337, 337, 337, 337, /* 2770 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, /* 2780 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, /* 2790 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, @@ -803,88 +803,88 @@ static const YYCODETYPE yy_lookahead[] = { }; #define YY_SHIFT_COUNT (791) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2417) +#define YY_SHIFT_MAX (2418) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 1097, 0, 104, 0, 336, 336, 336, 336, 336, 336, /* 10 */ 336, 336, 336, 336, 336, 336, 440, 671, 671, 775, /* 20 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, /* 30 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, /* 40 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 50 */ 671, 173, 483, 524, 446, 426, 476, 523, 476, 446, - /* 60 */ 446, 1111, 476, 1111, 1111, 275, 476, 141, 644, 223, + /* 50 */ 671, 173, 386, 524, 402, 426, 285, 337, 285, 402, + /* 60 */ 402, 1111, 285, 1111, 1111, 275, 285, 141, 644, 223, /* 70 */ 223, 644, 496, 496, 413, 33, 293, 293, 223, 223, - /* 80 */ 223, 223, 223, 223, 223, 410, 223, 223, 205, 141, - /* 90 */ 223, 223, 458, 223, 141, 223, 410, 223, 410, 141, - /* 100 */ 223, 223, 141, 223, 141, 141, 141, 223, 537, 917, + /* 80 */ 223, 223, 223, 223, 223, 446, 223, 223, 430, 141, + /* 90 */ 223, 223, 512, 223, 141, 223, 446, 223, 446, 141, + /* 100 */ 223, 223, 141, 223, 141, 141, 141, 223, 458, 917, /* 110 */ 15, 15, 565, 170, 953, 953, 953, 953, 953, 953, /* 120 */ 953, 953, 953, 953, 953, 953, 953, 953, 953, 953, - /* 130 */ 953, 953, 953, 1095, 14, 413, 33, 994, 994, 113, - /* 140 */ 116, 116, 116, 180, 266, 266, 113, 505, 505, 505, - /* 150 */ 205, 510, 513, 141, 697, 141, 697, 697, 713, 711, + /* 130 */ 953, 953, 953, 1095, 14, 413, 33, 932, 932, 113, + /* 140 */ 116, 116, 116, 180, 395, 395, 113, 529, 529, 529, + /* 150 */ 430, 487, 381, 141, 556, 141, 556, 556, 580, 677, /* 160 */ 297, 297, 297, 297, 297, 297, 297, 297, 1866, 2242, - /* 170 */ 185, 31, 508, 282, 271, 281, 341, 341, 417, 720, - /* 180 */ 330, 517, 1230, 826, 913, 970, 1008, 243, 352, 1008, - /* 190 */ 1122, 1146, 1000, 1258, 1474, 1508, 1326, 205, 1508, 205, - /* 200 */ 1350, 1505, 1529, 1505, 1383, 1536, 1536, 1505, 1383, 1383, - /* 210 */ 1473, 1477, 1536, 1481, 1536, 1536, 1536, 1566, 1540, 1566, - /* 220 */ 1540, 1508, 205, 1574, 205, 1578, 1580, 205, 1578, 205, - /* 230 */ 205, 205, 1536, 205, 1566, 141, 141, 141, 141, 141, - /* 240 */ 141, 141, 141, 141, 141, 141, 1536, 1566, 697, 697, - /* 250 */ 697, 1416, 1526, 1508, 537, 1435, 1438, 1574, 537, 1471, - /* 260 */ 1536, 1529, 1529, 697, 1414, 1409, 697, 1414, 1409, 697, - /* 270 */ 697, 141, 1411, 1518, 1414, 1426, 1428, 1446, 1258, 1418, - /* 280 */ 1430, 1434, 1452, 505, 1698, 1536, 1578, 537, 537, 1720, - /* 290 */ 1409, 697, 697, 697, 697, 697, 1409, 697, 1581, 537, - /* 300 */ 713, 537, 505, 1676, 1679, 697, 711, 1536, 537, 1750, - /* 310 */ 1566, 2716, 2716, 2716, 2716, 2716, 2716, 2716, 2716, 2716, - /* 320 */ 1018, 425, 224, 717, 306, 378, 1139, 718, 455, 871, - /* 330 */ 1182, 824, 971, 971, 971, 971, 971, 971, 971, 971, - /* 340 */ 971, 357, 1220, 118, 118, 451, 552, 556, 657, 76, - /* 350 */ 772, 954, 731, 664, 494, 494, 924, 1191, 636, 924, - /* 360 */ 924, 924, 645, 156, 821, 836, 1198, 1091, 362, 1153, - /* 370 */ 1181, 1192, 1197, 1223, 1277, 1309, 1335, 1340, 1115, 1297, - /* 380 */ 1314, 1120, 1324, 1325, 1327, 1165, 1130, 1037, 1244, 1328, - /* 390 */ 1330, 1331, 1356, 1363, 1364, 1408, 1370, 1142, 1382, 1355, - /* 400 */ 1389, 1398, 1410, 1412, 1420, 1425, 1286, 617, 1385, 1417, - /* 410 */ 1476, 1437, 1421, 1798, 1804, 1806, 1765, 1808, 1774, 1596, - /* 420 */ 1777, 1778, 1780, 1602, 1818, 1785, 1786, 1607, 1823, 1610, - /* 430 */ 1826, 1792, 1828, 1807, 1830, 1797, 1625, 1834, 1638, 1837, - /* 440 */ 1641, 1642, 1648, 1652, 1844, 1845, 1846, 1661, 1664, 1853, - /* 450 */ 1860, 1815, 1864, 1869, 1870, 1824, 1877, 1879, 1880, 1881, - /* 460 */ 1882, 1883, 1886, 1888, 1730, 1861, 1900, 1742, 1902, 1903, - /* 470 */ 1904, 1905, 1906, 1907, 1909, 1910, 1911, 1912, 1913, 1914, - /* 480 */ 1927, 1929, 1930, 1932, 1874, 1917, 1918, 1919, 1920, 1921, - /* 490 */ 1922, 1915, 1933, 1934, 1935, 1795, 1939, 1940, 1923, 1894, - /* 500 */ 1924, 1895, 1944, 1885, 1925, 1949, 1889, 1952, 1891, 1954, - /* 510 */ 1955, 1926, 1908, 1928, 1956, 1931, 1938, 1936, 1958, 1941, - /* 520 */ 1945, 1947, 1959, 1942, 1962, 1957, 1967, 1953, 1951, 1960, - /* 530 */ 1965, 1961, 1963, 1969, 1964, 1970, 1972, 1973, 1974, 1784, - /* 540 */ 1978, 1980, 1984, 1943, 1996, 2001, 1976, 1968, 1966, 2019, - /* 550 */ 1985, 1981, 1988, 2029, 1997, 1982, 1995, 2035, 2002, 1987, - /* 560 */ 2003, 2040, 2041, 2045, 2046, 2047, 2049, 1948, 1971, 2018, - /* 570 */ 2043, 2063, 2031, 2032, 2033, 2034, 2038, 2039, 2042, 2026, - /* 580 */ 2036, 2044, 2051, 2054, 2061, 2075, 2056, 2082, 2066, 2048, - /* 590 */ 2089, 2076, 2067, 2103, 2069, 2105, 2071, 2107, 2079, 2088, - /* 600 */ 2074, 2078, 2080, 2006, 2008, 2116, 1946, 2016, 1950, 2086, - /* 610 */ 2109, 2129, 1975, 2111, 1986, 1989, 2134, 2135, 1990, 1991, - /* 620 */ 2133, 2093, 1875, 2037, 2057, 2062, 2091, 2065, 2094, 2068, - /* 630 */ 2060, 2099, 2100, 2072, 2070, 2077, 2081, 2073, 2104, 2098, - /* 640 */ 2122, 2083, 2106, 1977, 2090, 2092, 2168, 2131, 1992, 2143, - /* 650 */ 2146, 2147, 2148, 2150, 2153, 2096, 2097, 2140, 1993, 2151, - /* 660 */ 2145, 2190, 2197, 2198, 2202, 2102, 2164, 1951, 2157, 2108, - /* 670 */ 2117, 2118, 2120, 2121, 2055, 2123, 2207, 2171, 2064, 2126, - /* 680 */ 2130, 1951, 2201, 2176, 2144, 2000, 2156, 2245, 2227, 2030, - /* 690 */ 2158, 2149, 2159, 2155, 2160, 2161, 2204, 2165, 2166, 2217, - /* 700 */ 2163, 2232, 2050, 2167, 2174, 2181, 2236, 2237, 2172, 2185, - /* 710 */ 2243, 2187, 2189, 2256, 2193, 2203, 2262, 2206, 2212, 2269, - /* 720 */ 2208, 2213, 2275, 2215, 2199, 2209, 2210, 2214, 2219, 2279, - /* 730 */ 2223, 2293, 2234, 2279, 2279, 2316, 2272, 2274, 2309, 2317, - /* 740 */ 2319, 2321, 2322, 2323, 2324, 2325, 2287, 2268, 2331, 2341, - /* 750 */ 2342, 2343, 2344, 2345, 2346, 2347, 2304, 2026, 2349, 2036, - /* 760 */ 2350, 2351, 2352, 2353, 2367, 2355, 2391, 2357, 2356, 2359, - /* 770 */ 2394, 2361, 2358, 2360, 2400, 2369, 2362, 2363, 2406, 2373, - /* 780 */ 2365, 2370, 2410, 2377, 2381, 2417, 2396, 2398, 2399, 2401, - /* 790 */ 2404, 2402, + /* 170 */ 185, 31, 508, 282, 271, 281, 341, 341, 417, 1035, + /* 180 */ 785, 262, 1261, 1136, 739, 1192, 1042, 993, 1090, 1042, + /* 190 */ 1004, 950, 1205, 1257, 1470, 1500, 1325, 430, 1500, 430, + /* 200 */ 1349, 1503, 1528, 1503, 1382, 1535, 1535, 1503, 1382, 1382, + /* 210 */ 1475, 1472, 1535, 1484, 1535, 1535, 1535, 1567, 1541, 1567, + /* 220 */ 1541, 1500, 430, 1575, 430, 1580, 1582, 430, 1580, 430, + /* 230 */ 430, 430, 1535, 430, 1567, 141, 141, 141, 141, 141, + /* 240 */ 141, 141, 141, 141, 141, 141, 1535, 1567, 556, 556, + /* 250 */ 556, 1416, 1530, 1500, 458, 1438, 1442, 1575, 458, 1459, + /* 260 */ 1535, 1528, 1528, 556, 1400, 1422, 556, 1400, 1422, 556, + /* 270 */ 556, 141, 1423, 1519, 1400, 1426, 1428, 1450, 1257, 1418, + /* 280 */ 1431, 1437, 1461, 529, 1698, 1535, 1580, 458, 458, 1707, + /* 290 */ 1422, 556, 556, 556, 556, 556, 1422, 556, 1570, 458, + /* 300 */ 580, 458, 529, 1657, 1679, 556, 677, 1535, 458, 1750, + /* 310 */ 1567, 2716, 2716, 2716, 2716, 2716, 2716, 2716, 2716, 2716, + /* 320 */ 1018, 425, 224, 306, 717, 1139, 1156, 257, 371, 455, + /* 330 */ 871, 718, 1176, 1176, 1176, 1176, 1176, 1176, 1176, 1176, + /* 340 */ 1176, 357, 1155, 118, 118, 714, 680, 187, 1050, 76, + /* 350 */ 716, 954, 1229, 822, 494, 494, 852, 844, 488, 852, + /* 360 */ 852, 852, 156, 1020, 1268, 237, 1098, 1181, 1340, 1251, + /* 370 */ 1264, 1275, 1282, 1296, 1307, 1365, 1393, 1397, 1038, 1323, + /* 380 */ 1324, 1297, 1354, 1356, 1360, 1266, 1012, 1209, 1254, 1364, + /* 390 */ 1368, 1370, 1378, 1385, 1387, 1408, 1389, 1189, 1398, 327, + /* 400 */ 1410, 1412, 1425, 1427, 1452, 1471, 1317, 1331, 1381, 1386, + /* 410 */ 1404, 1366, 1434, 1798, 1799, 1805, 1764, 1809, 1776, 1597, + /* 420 */ 1778, 1780, 1783, 1602, 1820, 1786, 1787, 1608, 1825, 1611, + /* 430 */ 1827, 1793, 1829, 1808, 1832, 1800, 1626, 1834, 1639, 1839, + /* 440 */ 1642, 1643, 1649, 1654, 1845, 1846, 1847, 1663, 1668, 1860, + /* 450 */ 1862, 1817, 1869, 1870, 1877, 1824, 1879, 1880, 1881, 1882, + /* 460 */ 1883, 1886, 1888, 1889, 1737, 1865, 1901, 1743, 1903, 1904, + /* 470 */ 1905, 1906, 1907, 1909, 1910, 1911, 1912, 1913, 1914, 1927, + /* 480 */ 1929, 1930, 1932, 1933, 1874, 1917, 1918, 1919, 1920, 1921, + /* 490 */ 1922, 1915, 1934, 1935, 1938, 1796, 1940, 1942, 1923, 1895, + /* 500 */ 1924, 1896, 1947, 1887, 1925, 1951, 1890, 1953, 1892, 1955, + /* 510 */ 1956, 1926, 1908, 1928, 1958, 1931, 1945, 1936, 1959, 1941, + /* 520 */ 1948, 1939, 1962, 1944, 1963, 1954, 1960, 1943, 1961, 1964, + /* 530 */ 1970, 1965, 1972, 1946, 1967, 1973, 1977, 1980, 1968, 1784, + /* 540 */ 1998, 2001, 2003, 1950, 2009, 2013, 1982, 1978, 1966, 2020, + /* 550 */ 1994, 1981, 1992, 2032, 1999, 1986, 1997, 2037, 2006, 1991, + /* 560 */ 2007, 2042, 2045, 2047, 2049, 2060, 2063, 1858, 1957, 2018, + /* 570 */ 2043, 2066, 2033, 2034, 2035, 2038, 2039, 2040, 2041, 2036, + /* 580 */ 2044, 2046, 2051, 2055, 2054, 2078, 2057, 2082, 2074, 2048, + /* 590 */ 2098, 2079, 2067, 2103, 2069, 2105, 2071, 2107, 2086, 2089, + /* 600 */ 2075, 2076, 2080, 2012, 2010, 2118, 1949, 2016, 1969, 2094, + /* 610 */ 2109, 2133, 1952, 2112, 1971, 1974, 2135, 2136, 1975, 1979, + /* 620 */ 2134, 2095, 1875, 2058, 2059, 2061, 2092, 2064, 2096, 2068, + /* 630 */ 2065, 2106, 2124, 2072, 2070, 2077, 2081, 2073, 2126, 2115, + /* 640 */ 2128, 2083, 2127, 1984, 2088, 2090, 2139, 2137, 1985, 2143, + /* 650 */ 2147, 2148, 2150, 2153, 2155, 2091, 2093, 2140, 1983, 2154, + /* 660 */ 2149, 2207, 2210, 2211, 2220, 2097, 2162, 1961, 2156, 2099, + /* 670 */ 2100, 2117, 2102, 2119, 2062, 2120, 2224, 2187, 2056, 2141, + /* 680 */ 2130, 1961, 2183, 2201, 2151, 2011, 2158, 2247, 2228, 2029, + /* 690 */ 2159, 2157, 2160, 2161, 2164, 2165, 2206, 2167, 2168, 2217, + /* 700 */ 2170, 2232, 2085, 2169, 2171, 2174, 2222, 2233, 2172, 2184, + /* 710 */ 2254, 2188, 2186, 2258, 2193, 2203, 2262, 2208, 2212, 2269, + /* 720 */ 2214, 2218, 2275, 2215, 2198, 2199, 2209, 2213, 2223, 2284, + /* 730 */ 2234, 2300, 2236, 2284, 2284, 2319, 2273, 2276, 2317, 2321, + /* 740 */ 2322, 2323, 2324, 2325, 2327, 2330, 2279, 2280, 2331, 2341, + /* 750 */ 2343, 2344, 2358, 2346, 2347, 2349, 2291, 2036, 2350, 2044, + /* 760 */ 2351, 2352, 2353, 2354, 2368, 2356, 2392, 2359, 2355, 2357, + /* 770 */ 2398, 2364, 2360, 2361, 2402, 2370, 2362, 2367, 2407, 2373, + /* 780 */ 2363, 2371, 2414, 2381, 2382, 2418, 2397, 2399, 2400, 2401, + /* 790 */ 2404, 2406, }; #define YY_REDUCE_COUNT (319) #define YY_REDUCE_MIN (-454) @@ -895,115 +895,115 @@ static const short yy_reduce_ofst[] = { /* 20 */ 499, 1240, 1302, 837, 1319, 1338, 1424, 1441, 1460, 1558, /* 30 */ 1591, 1624, 1660, 1693, 1722, 1782, 1801, 1849, 1868, 1916, /* 40 */ 1937, 2005, 2023, 2053, 2084, 2101, 2152, 2173, 2200, 2261, - /* 50 */ 2278, 301, 1227, -402, -167, -360, 252, 709, 803, 465, - /* 60 */ 674, -348, 886, 259, 819, -454, -374, -322, -8, -227, + /* 50 */ 2278, 301, 1227, -402, -167, -360, 252, 483, 505, 465, + /* 60 */ 674, -348, 803, 259, 886, -454, -374, -322, 205, -227, /* 70 */ -72, -249, -289, 61, -362, 332, -284, -238, -204, -41, - /* 80 */ 130, 169, 265, 272, 338, -214, 379, 394, 97, -5, - /* 90 */ 495, 622, 29, 672, 450, 767, -182, 809, -144, 119, - /* 100 */ 944, 1035, 481, 1043, 344, 563, 392, 1048, 78, -308, - /* 110 */ -442, -442, -358, -333, -321, -215, 204, 307, 498, 574, - /* 120 */ 584, 587, 588, 609, 642, 654, 703, 747, 779, 781, - /* 130 */ 782, 817, 840, -301, -70, -4, 346, 701, 705, 662, - /* 140 */ -70, 296, 335, 84, 197, 419, 722, 136, 164, 806, - /* 150 */ 850, -293, 774, -312, 854, 576, 861, 862, 879, 889, - /* 160 */ -364, 497, 502, 516, 650, 746, 761, 650, 528, 851, - /* 170 */ 948, 900, 884, 910, 1034, 949, 1075, 1087, 1061, 1061, - /* 180 */ 1113, 1082, 1136, 1141, 1103, 1096, 1040, 1040, 1033, 1040, - /* 190 */ 1065, 1058, 1061, 1094, 1101, 1112, 1114, 1174, 1116, 1177, - /* 200 */ 1134, 1151, 1150, 1154, 1160, 1211, 1212, 1166, 1176, 1178, - /* 210 */ 1213, 1217, 1232, 1222, 1234, 1235, 1236, 1245, 1243, 1249, - /* 220 */ 1247, 1179, 1237, 1204, 1241, 1250, 1193, 1251, 1256, 1252, - /* 230 */ 1253, 1255, 1259, 1257, 1260, 1246, 1248, 1261, 1262, 1265, - /* 240 */ 1272, 1275, 1276, 1278, 1279, 1282, 1270, 1280, 1224, 1231, - /* 250 */ 1238, 1205, 1215, 1216, 1281, 1219, 1239, 1269, 1306, 1264, - /* 260 */ 1317, 1268, 1284, 1291, 1225, 1288, 1299, 1226, 1290, 1308, - /* 270 */ 1310, 1061, 1228, 1271, 1254, 1263, 1285, 1283, 1287, 1233, - /* 280 */ 1242, 1267, 1040, 1365, 1293, 1374, 1373, 1369, 1375, 1339, - /* 290 */ 1336, 1357, 1367, 1368, 1371, 1376, 1354, 1377, 1358, 1407, - /* 300 */ 1402, 1422, 1427, 1333, 1401, 1394, 1419, 1436, 1432, 1447, - /* 310 */ 1450, 1381, 1378, 1388, 1390, 1423, 1431, 1433, 1442, 1451, + /* 80 */ 130, 169, 265, 272, 338, -214, 393, 475, 97, 201, + /* 90 */ 591, 622, 493, 672, 385, 767, -182, 809, 58, 344, + /* 100 */ 833, 845, 480, 848, 378, 856, 604, 906, 409, -308, + /* 110 */ -442, -442, -358, -333, -321, -215, 111, 307, 500, 534, + /* 120 */ 548, 587, 588, 609, 654, 747, 817, 840, 868, 902, + /* 130 */ 914, 951, 965, -301, 80, 87, 376, 473, 705, 722, + /* 140 */ 80, 197, 380, 122, 421, 678, 925, 164, 339, 633, + /* 150 */ 153, -293, -88, -312, -28, 226, 382, 735, 354, 828, + /* 160 */ -364, 407, 415, 502, 516, 590, 702, 516, 545, 751, + /* 170 */ 825, 774, 757, 727, 968, 782, 969, 1052, 1047, 1047, + /* 180 */ 1079, 1053, 1087, 1102, 1077, 1067, 1028, 1028, 1022, 1028, + /* 190 */ 1059, 1051, 1047, 1086, 1099, 1108, 1105, 1174, 1116, 1177, + /* 200 */ 1127, 1150, 1149, 1153, 1157, 1211, 1212, 1166, 1179, 1182, + /* 210 */ 1216, 1218, 1232, 1223, 1235, 1236, 1237, 1246, 1245, 1252, + /* 220 */ 1247, 1178, 1239, 1206, 1241, 1250, 1193, 1249, 1258, 1253, + /* 230 */ 1255, 1256, 1259, 1263, 1273, 1244, 1248, 1262, 1265, 1272, + /* 240 */ 1276, 1277, 1278, 1279, 1283, 1284, 1270, 1285, 1220, 1234, + /* 250 */ 1238, 1207, 1215, 1213, 1280, 1222, 1260, 1274, 1304, 1267, + /* 260 */ 1316, 1281, 1287, 1289, 1219, 1286, 1300, 1228, 1290, 1306, + /* 270 */ 1308, 1047, 1242, 1271, 1243, 1291, 1294, 1269, 1288, 1231, + /* 280 */ 1298, 1301, 1028, 1372, 1295, 1371, 1374, 1367, 1384, 1339, + /* 290 */ 1336, 1355, 1357, 1358, 1369, 1373, 1353, 1376, 1359, 1411, + /* 300 */ 1403, 1429, 1430, 1332, 1401, 1394, 1419, 1439, 1433, 1447, + /* 310 */ 1451, 1380, 1375, 1390, 1391, 1432, 1435, 1436, 1446, 1457, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 10 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 20 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 30 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 40 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 50 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 60 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 70 */ 1764, 1764, 1764, 1764, 2045, 1764, 1764, 1764, 1764, 1764, - /* 80 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1853, 1764, - /* 90 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 100 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1851, 2038, - /* 110 */ 2263, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 120 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 130 */ 1764, 1764, 1764, 1764, 2275, 1764, 1764, 1827, 1827, 1764, - /* 140 */ 2275, 2275, 2275, 1851, 2235, 2235, 1764, 1764, 1764, 1764, - /* 150 */ 1853, 2105, 1764, 1764, 1764, 1764, 1764, 1764, 1973, 1764, - /* 160 */ 1764, 1764, 1764, 1764, 1997, 1764, 1764, 1764, 2097, 1764, - /* 170 */ 1764, 2300, 2356, 1764, 1764, 2303, 1764, 1764, 1764, 1764, - /* 180 */ 1764, 2050, 1764, 1764, 1926, 2290, 2267, 2281, 2340, 2268, - /* 190 */ 2265, 2284, 1764, 2294, 1764, 1764, 2119, 1853, 1764, 1853, - /* 200 */ 2084, 2043, 1764, 2043, 2040, 1764, 1764, 2043, 2040, 2040, - /* 210 */ 1915, 1911, 1764, 1909, 1764, 1764, 1764, 1764, 1811, 1764, - /* 220 */ 1811, 1764, 1853, 1764, 1853, 1764, 1764, 1853, 1764, 1853, - /* 230 */ 1853, 1853, 1764, 1853, 1764, 1764, 1764, 1764, 1764, 1764, - /* 240 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 250 */ 1764, 2117, 2103, 1764, 1851, 2095, 2093, 1764, 1851, 2091, - /* 260 */ 1764, 1764, 1764, 1764, 2311, 2309, 1764, 2311, 2309, 1764, - /* 270 */ 1764, 1764, 2325, 2321, 2311, 2329, 2327, 2296, 2294, 2359, - /* 280 */ 2346, 2342, 2281, 1764, 1764, 1764, 1764, 1851, 1851, 1764, - /* 290 */ 2309, 1764, 1764, 1764, 1764, 1764, 2309, 1764, 1764, 1851, - /* 300 */ 1764, 1851, 1764, 1764, 1942, 1764, 1764, 1764, 1851, 1796, - /* 310 */ 1764, 2086, 2108, 2068, 2068, 1976, 1976, 1976, 1854, 1769, - /* 320 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 330 */ 1764, 1764, 2324, 2323, 2190, 1764, 2239, 2238, 2237, 2228, - /* 340 */ 2189, 1938, 1764, 2188, 2187, 1764, 1764, 1764, 1764, 1764, - /* 350 */ 1764, 1764, 1764, 1764, 2059, 2058, 2181, 1764, 1764, 2182, - /* 360 */ 2180, 2179, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 370 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 380 */ 1764, 1764, 1764, 1764, 1764, 1764, 2343, 2347, 1764, 1764, - /* 390 */ 1764, 1764, 1764, 1764, 1764, 2264, 1764, 1764, 1764, 2163, - /* 400 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 410 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 420 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 430 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 440 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 450 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 460 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 470 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 480 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 490 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 500 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 510 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 520 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1801, 2168, 1764, - /* 530 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 540 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 550 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 560 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 570 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1892, - /* 580 */ 1891, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 590 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 600 */ 1764, 1764, 1764, 2172, 1764, 1764, 1764, 1764, 1764, 1764, - /* 610 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 620 */ 2339, 2297, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 630 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 640 */ 2163, 1764, 2322, 1764, 1764, 2337, 1764, 2341, 1764, 1764, - /* 650 */ 1764, 1764, 1764, 1764, 1764, 2274, 2270, 1764, 1764, 2266, - /* 660 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 2171, 1764, 1764, - /* 670 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 680 */ 1764, 2162, 1764, 2225, 1764, 1764, 1764, 2259, 1764, 1764, - /* 690 */ 2210, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 700 */ 2172, 1764, 2175, 1764, 1764, 1764, 1764, 1764, 1970, 1764, - /* 710 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 720 */ 1764, 1764, 1764, 1764, 1954, 1952, 1951, 1950, 1764, 1983, - /* 730 */ 1764, 1764, 1764, 1979, 1978, 1764, 1764, 1764, 1764, 1764, - /* 740 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1872, 1764, - /* 750 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1864, 1764, 1863, - /* 760 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 770 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 780 */ 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, 1764, - /* 790 */ 1764, 1764, + /* 0 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 10 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 20 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 30 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 40 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 50 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 60 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 70 */ 1766, 1766, 1766, 1766, 2047, 1766, 1766, 1766, 1766, 1766, + /* 80 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1855, 1766, + /* 90 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 100 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1853, 2040, + /* 110 */ 2265, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 120 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 130 */ 1766, 1766, 1766, 1766, 2277, 1766, 1766, 1829, 1829, 1766, + /* 140 */ 2277, 2277, 2277, 1853, 2237, 2237, 1766, 1766, 1766, 1766, + /* 150 */ 1855, 2107, 1766, 1766, 1766, 1766, 1766, 1766, 1975, 1766, + /* 160 */ 1766, 1766, 1766, 1766, 1999, 1766, 1766, 1766, 2099, 1766, + /* 170 */ 1766, 2302, 2359, 1766, 1766, 2305, 1766, 1766, 1766, 1766, + /* 180 */ 1766, 2052, 1766, 1766, 1928, 2292, 2269, 2283, 2343, 2270, + /* 190 */ 2267, 2286, 1766, 2296, 1766, 1766, 2121, 1855, 1766, 1855, + /* 200 */ 2086, 2045, 1766, 2045, 2042, 1766, 1766, 2045, 2042, 2042, + /* 210 */ 1917, 1913, 1766, 1911, 1766, 1766, 1766, 1766, 1813, 1766, + /* 220 */ 1813, 1766, 1855, 1766, 1855, 1766, 1766, 1855, 1766, 1855, + /* 230 */ 1855, 1855, 1766, 1855, 1766, 1766, 1766, 1766, 1766, 1766, + /* 240 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 250 */ 1766, 2119, 2105, 1766, 1853, 2097, 2095, 1766, 1853, 2093, + /* 260 */ 1766, 1766, 1766, 1766, 2313, 2311, 1766, 2313, 2311, 1766, + /* 270 */ 1766, 1766, 2327, 2323, 2313, 2332, 2329, 2298, 2296, 2362, + /* 280 */ 2349, 2345, 2283, 1766, 1766, 1766, 1766, 1853, 1853, 1766, + /* 290 */ 2311, 1766, 1766, 1766, 1766, 1766, 2311, 1766, 1766, 1853, + /* 300 */ 1766, 1853, 1766, 1766, 1944, 1766, 1766, 1766, 1853, 1798, + /* 310 */ 1766, 2088, 2110, 2070, 2070, 1978, 1978, 1978, 1856, 1771, + /* 320 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 330 */ 1766, 1766, 2326, 2325, 2192, 1766, 2241, 2240, 2239, 2230, + /* 340 */ 2191, 1940, 1766, 2190, 2189, 1766, 1766, 1766, 1766, 1766, + /* 350 */ 1766, 1766, 1766, 1766, 2061, 2060, 2183, 1766, 1766, 2184, + /* 360 */ 2182, 2181, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 370 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 380 */ 1766, 1766, 1766, 1766, 1766, 1766, 2346, 2350, 1766, 1766, + /* 390 */ 1766, 1766, 1766, 1766, 1766, 2266, 1766, 1766, 1766, 2165, + /* 400 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 410 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 420 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 430 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 440 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 450 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 460 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 470 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 480 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 490 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 500 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 510 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 520 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1803, 2170, 1766, + /* 530 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 540 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 550 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 560 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 570 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1894, + /* 580 */ 1893, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 590 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 600 */ 1766, 1766, 1766, 2174, 1766, 1766, 1766, 1766, 1766, 1766, + /* 610 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 620 */ 2342, 2299, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 630 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 640 */ 2165, 1766, 2324, 1766, 1766, 2340, 1766, 2344, 1766, 1766, + /* 650 */ 1766, 1766, 1766, 1766, 1766, 2276, 2272, 1766, 1766, 2268, + /* 660 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 2173, 1766, 1766, + /* 670 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 680 */ 1766, 2164, 1766, 2227, 1766, 1766, 1766, 2261, 1766, 1766, + /* 690 */ 2212, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 700 */ 2174, 1766, 2177, 1766, 1766, 1766, 1766, 1766, 1972, 1766, + /* 710 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 720 */ 1766, 1766, 1766, 1766, 1956, 1954, 1953, 1952, 1766, 1985, + /* 730 */ 1766, 1766, 1766, 1981, 1980, 1766, 1766, 1766, 1766, 1766, + /* 740 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1874, 1766, + /* 750 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1866, 1766, 1865, + /* 760 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 770 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 780 */ 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, 1766, + /* 790 */ 1766, 1766, }; /********** End of lemon-generated parsing tables *****************************/ @@ -2501,39 +2501,40 @@ static const char *const yyRuleName[] = { /* 559 */ "having_clause_opt ::= HAVING search_condition", /* 560 */ "range_opt ::=", /* 561 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 562 */ "every_opt ::=", - /* 563 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 564 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 565 */ "query_simple ::= query_specification", - /* 566 */ "query_simple ::= union_query_expression", - /* 567 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 568 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 569 */ "query_simple_or_subquery ::= query_simple", - /* 570 */ "query_simple_or_subquery ::= subquery", - /* 571 */ "query_or_subquery ::= query_expression", - /* 572 */ "query_or_subquery ::= subquery", - /* 573 */ "order_by_clause_opt ::=", - /* 574 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 575 */ "slimit_clause_opt ::=", - /* 576 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 577 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 578 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 579 */ "limit_clause_opt ::=", - /* 580 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 581 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 582 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 583 */ "subquery ::= NK_LP query_expression NK_RP", - /* 584 */ "subquery ::= NK_LP subquery NK_RP", - /* 585 */ "search_condition ::= common_expression", - /* 586 */ "sort_specification_list ::= sort_specification", - /* 587 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 588 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 589 */ "ordering_specification_opt ::=", - /* 590 */ "ordering_specification_opt ::= ASC", - /* 591 */ "ordering_specification_opt ::= DESC", - /* 592 */ "null_ordering_opt ::=", - /* 593 */ "null_ordering_opt ::= NULLS FIRST", - /* 594 */ "null_ordering_opt ::= NULLS LAST", + /* 562 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_RP", + /* 563 */ "every_opt ::=", + /* 564 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 565 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 566 */ "query_simple ::= query_specification", + /* 567 */ "query_simple ::= union_query_expression", + /* 568 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 569 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 570 */ "query_simple_or_subquery ::= query_simple", + /* 571 */ "query_simple_or_subquery ::= subquery", + /* 572 */ "query_or_subquery ::= query_expression", + /* 573 */ "query_or_subquery ::= subquery", + /* 574 */ "order_by_clause_opt ::=", + /* 575 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 576 */ "slimit_clause_opt ::=", + /* 577 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 578 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 579 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 580 */ "limit_clause_opt ::=", + /* 581 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 582 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 583 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 584 */ "subquery ::= NK_LP query_expression NK_RP", + /* 585 */ "subquery ::= NK_LP subquery NK_RP", + /* 586 */ "search_condition ::= common_expression", + /* 587 */ "sort_specification_list ::= sort_specification", + /* 588 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 589 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 590 */ "ordering_specification_opt ::=", + /* 591 */ "ordering_specification_opt ::= ASC", + /* 592 */ "ordering_specification_opt ::= DESC", + /* 593 */ "null_ordering_opt ::=", + /* 594 */ "null_ordering_opt ::= NULLS FIRST", + /* 595 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -3716,39 +3717,40 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 468, /* (559) having_clause_opt ::= HAVING search_condition */ 463, /* (560) range_opt ::= */ 463, /* (561) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - 464, /* (562) every_opt ::= */ - 464, /* (563) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - 474, /* (564) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - 475, /* (565) query_simple ::= query_specification */ - 475, /* (566) query_simple ::= union_query_expression */ - 479, /* (567) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - 479, /* (568) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - 480, /* (569) query_simple_or_subquery ::= query_simple */ - 480, /* (570) query_simple_or_subquery ::= subquery */ - 409, /* (571) query_or_subquery ::= query_expression */ - 409, /* (572) query_or_subquery ::= subquery */ - 476, /* (573) order_by_clause_opt ::= */ - 476, /* (574) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 477, /* (575) slimit_clause_opt ::= */ - 477, /* (576) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - 477, /* (577) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - 477, /* (578) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 478, /* (579) limit_clause_opt ::= */ - 478, /* (580) limit_clause_opt ::= LIMIT NK_INTEGER */ - 478, /* (581) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - 478, /* (582) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 456, /* (583) subquery ::= NK_LP query_expression NK_RP */ - 456, /* (584) subquery ::= NK_LP subquery NK_RP */ - 352, /* (585) search_condition ::= common_expression */ - 481, /* (586) sort_specification_list ::= sort_specification */ - 481, /* (587) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - 482, /* (588) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 483, /* (589) ordering_specification_opt ::= */ - 483, /* (590) ordering_specification_opt ::= ASC */ - 483, /* (591) ordering_specification_opt ::= DESC */ - 484, /* (592) null_ordering_opt ::= */ - 484, /* (593) null_ordering_opt ::= NULLS FIRST */ - 484, /* (594) null_ordering_opt ::= NULLS LAST */ + 463, /* (562) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 464, /* (563) every_opt ::= */ + 464, /* (564) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + 474, /* (565) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + 475, /* (566) query_simple ::= query_specification */ + 475, /* (567) query_simple ::= union_query_expression */ + 479, /* (568) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + 479, /* (569) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + 480, /* (570) query_simple_or_subquery ::= query_simple */ + 480, /* (571) query_simple_or_subquery ::= subquery */ + 409, /* (572) query_or_subquery ::= query_expression */ + 409, /* (573) query_or_subquery ::= subquery */ + 476, /* (574) order_by_clause_opt ::= */ + 476, /* (575) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 477, /* (576) slimit_clause_opt ::= */ + 477, /* (577) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + 477, /* (578) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + 477, /* (579) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 478, /* (580) limit_clause_opt ::= */ + 478, /* (581) limit_clause_opt ::= LIMIT NK_INTEGER */ + 478, /* (582) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + 478, /* (583) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 456, /* (584) subquery ::= NK_LP query_expression NK_RP */ + 456, /* (585) subquery ::= NK_LP subquery NK_RP */ + 352, /* (586) search_condition ::= common_expression */ + 481, /* (587) sort_specification_list ::= sort_specification */ + 481, /* (588) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + 482, /* (589) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 483, /* (590) ordering_specification_opt ::= */ + 483, /* (591) ordering_specification_opt ::= ASC */ + 483, /* (592) ordering_specification_opt ::= DESC */ + 484, /* (593) null_ordering_opt ::= */ + 484, /* (594) null_ordering_opt ::= NULLS FIRST */ + 484, /* (595) null_ordering_opt ::= NULLS LAST */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -4316,39 +4318,40 @@ static const signed char yyRuleInfoNRhs[] = { -2, /* (559) having_clause_opt ::= HAVING search_condition */ 0, /* (560) range_opt ::= */ -6, /* (561) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - 0, /* (562) every_opt ::= */ - -4, /* (563) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - -4, /* (564) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - -1, /* (565) query_simple ::= query_specification */ - -1, /* (566) query_simple ::= union_query_expression */ - -4, /* (567) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - -3, /* (568) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - -1, /* (569) query_simple_or_subquery ::= query_simple */ - -1, /* (570) query_simple_or_subquery ::= subquery */ - -1, /* (571) query_or_subquery ::= query_expression */ - -1, /* (572) query_or_subquery ::= subquery */ - 0, /* (573) order_by_clause_opt ::= */ - -3, /* (574) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 0, /* (575) slimit_clause_opt ::= */ - -2, /* (576) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - -4, /* (577) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - -4, /* (578) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 0, /* (579) limit_clause_opt ::= */ - -2, /* (580) limit_clause_opt ::= LIMIT NK_INTEGER */ - -4, /* (581) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - -4, /* (582) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - -3, /* (583) subquery ::= NK_LP query_expression NK_RP */ - -3, /* (584) subquery ::= NK_LP subquery NK_RP */ - -1, /* (585) search_condition ::= common_expression */ - -1, /* (586) sort_specification_list ::= sort_specification */ - -3, /* (587) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - -3, /* (588) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 0, /* (589) ordering_specification_opt ::= */ - -1, /* (590) ordering_specification_opt ::= ASC */ - -1, /* (591) ordering_specification_opt ::= DESC */ - 0, /* (592) null_ordering_opt ::= */ - -2, /* (593) null_ordering_opt ::= NULLS FIRST */ - -2, /* (594) null_ordering_opt ::= NULLS LAST */ + -4, /* (562) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 0, /* (563) every_opt ::= */ + -4, /* (564) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + -4, /* (565) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + -1, /* (566) query_simple ::= query_specification */ + -1, /* (567) query_simple ::= union_query_expression */ + -4, /* (568) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + -3, /* (569) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + -1, /* (570) query_simple_or_subquery ::= query_simple */ + -1, /* (571) query_simple_or_subquery ::= subquery */ + -1, /* (572) query_or_subquery ::= query_expression */ + -1, /* (573) query_or_subquery ::= subquery */ + 0, /* (574) order_by_clause_opt ::= */ + -3, /* (575) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 0, /* (576) slimit_clause_opt ::= */ + -2, /* (577) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + -4, /* (578) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + -4, /* (579) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 0, /* (580) limit_clause_opt ::= */ + -2, /* (581) limit_clause_opt ::= LIMIT NK_INTEGER */ + -4, /* (582) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + -4, /* (583) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + -3, /* (584) subquery ::= NK_LP query_expression NK_RP */ + -3, /* (585) subquery ::= NK_LP subquery NK_RP */ + -1, /* (586) search_condition ::= common_expression */ + -1, /* (587) sort_specification_list ::= sort_specification */ + -3, /* (588) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + -3, /* (589) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 0, /* (590) ordering_specification_opt ::= */ + -1, /* (591) ordering_specification_opt ::= ASC */ + -1, /* (592) ordering_specification_opt ::= DESC */ + 0, /* (593) null_ordering_opt ::= */ + -2, /* (594) null_ordering_opt ::= NULLS FIRST */ + -2, /* (595) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -4564,9 +4567,9 @@ static YYACTIONTYPE yy_reduce( case 544: /* fill_opt ::= */ yytestcase(yyruleno==544); case 558: /* having_clause_opt ::= */ yytestcase(yyruleno==558); case 560: /* range_opt ::= */ yytestcase(yyruleno==560); - case 562: /* every_opt ::= */ yytestcase(yyruleno==562); - case 575: /* slimit_clause_opt ::= */ yytestcase(yyruleno==575); - case 579: /* limit_clause_opt ::= */ yytestcase(yyruleno==579); + case 563: /* every_opt ::= */ yytestcase(yyruleno==563); + case 576: /* slimit_clause_opt ::= */ yytestcase(yyruleno==576); + case 580: /* limit_clause_opt ::= */ yytestcase(yyruleno==580); { yymsp[1].minor.yy242 = NULL; } break; case 45: /* with_opt ::= WITH search_condition */ @@ -4941,7 +4944,7 @@ static YYACTIONTYPE yy_reduce( case 465: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==465); case 520: /* select_list ::= select_item */ yytestcase(yyruleno==520); case 531: /* partition_list ::= partition_item */ yytestcase(yyruleno==531); - case 586: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==586); + case 587: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==587); { yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy242); } yymsp[0].minor.yy174 = yylhsminor.yy174; break; @@ -4956,7 +4959,7 @@ static YYACTIONTYPE yy_reduce( case 460: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==460); case 521: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==521); case 532: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==532); - case 587: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==587); + case 588: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==588); { yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy242); } yymsp[-2].minor.yy174 = yylhsminor.yy174; break; @@ -5065,7 +5068,7 @@ static YYACTIONTYPE yy_reduce( case 338: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==338); case 529: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==529); case 554: /* group_by_clause_opt ::= */ yytestcase(yyruleno==554); - case 573: /* order_by_clause_opt ::= */ yytestcase(yyruleno==573); + case 574: /* order_by_clause_opt ::= */ yytestcase(yyruleno==574); { yymsp[1].minor.yy174 = NULL; } break; case 176: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ @@ -5526,7 +5529,7 @@ static YYACTIONTYPE yy_reduce( break; case 351: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ case 543: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==543); - case 563: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==563); + case 564: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==564); { yymsp[-3].minor.yy242 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy242); } break; case 354: /* cmd ::= KILL CONNECTION NK_INTEGER */ @@ -5602,10 +5605,10 @@ static YYACTIONTYPE yy_reduce( case 502: /* table_reference ::= table_primary */ yytestcase(yyruleno==502); case 503: /* table_reference ::= joined_table */ yytestcase(yyruleno==503); case 507: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==507); - case 565: /* query_simple ::= query_specification */ yytestcase(yyruleno==565); - case 566: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==566); - case 569: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==569); - case 571: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==571); + case 566: /* query_simple ::= query_specification */ yytestcase(yyruleno==566); + case 567: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==567); + case 570: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==570); + case 572: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==572); { yylhsminor.yy242 = yymsp[0].minor.yy242; } yymsp[0].minor.yy242 = yylhsminor.yy242; break; @@ -5667,9 +5670,9 @@ static YYACTIONTYPE yy_reduce( case 461: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==461); case 523: /* select_item ::= common_expression */ yytestcase(yyruleno==523); case 533: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==533); - case 570: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==570); - case 572: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==572); - case 585: /* search_condition ::= common_expression */ yytestcase(yyruleno==585); + case 571: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==571); + case 573: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==573); + case 586: /* search_condition ::= common_expression */ yytestcase(yyruleno==586); { yylhsminor.yy242 = releaseRawExprNode(pCxt, yymsp[0].minor.yy242); } yymsp[0].minor.yy242 = yylhsminor.yy242; break; @@ -5683,7 +5686,7 @@ static YYACTIONTYPE yy_reduce( break; case 411: /* expression ::= NK_LP expression NK_RP */ case 495: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==495); - case 584: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==584); + case 585: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==585); { yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } yymsp[-2].minor.yy242 = yylhsminor.yy242; break; @@ -6000,7 +6003,7 @@ static YYACTIONTYPE yy_reduce( break; case 530: /* partition_by_clause_opt ::= PARTITION BY partition_list */ case 555: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==555); - case 574: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==574); + case 575: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==575); { yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } break; case 537: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ @@ -6056,7 +6059,10 @@ static YYACTIONTYPE yy_reduce( case 561: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ { yymsp[-5].minor.yy242 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } break; - case 564: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 562: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy242 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } + break; + case 565: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { yylhsminor.yy242 = addOrderByClause(pCxt, yymsp[-3].minor.yy242, yymsp[-2].minor.yy174); yylhsminor.yy242 = addSlimitClause(pCxt, yylhsminor.yy242, yymsp[-1].minor.yy242); @@ -6064,50 +6070,50 @@ static YYACTIONTYPE yy_reduce( } yymsp[-3].minor.yy242 = yylhsminor.yy242; break; - case 567: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + case 568: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ { yylhsminor.yy242 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy242, yymsp[0].minor.yy242); } yymsp[-3].minor.yy242 = yylhsminor.yy242; break; - case 568: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + case 569: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ { yylhsminor.yy242 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy242, yymsp[0].minor.yy242); } yymsp[-2].minor.yy242 = yylhsminor.yy242; break; - case 576: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 580: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==580); + case 577: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 581: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==581); { yymsp[-1].minor.yy242 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 577: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 581: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==581); + case 578: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 582: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==582); { yymsp[-3].minor.yy242 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 578: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 582: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==582); + case 579: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 583: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==583); { yymsp[-3].minor.yy242 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 583: /* subquery ::= NK_LP query_expression NK_RP */ + case 584: /* subquery ::= NK_LP query_expression NK_RP */ { yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy242); } yymsp[-2].minor.yy242 = yylhsminor.yy242; break; - case 588: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + case 589: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ { yylhsminor.yy242 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), yymsp[-1].minor.yy48, yymsp[0].minor.yy687); } yymsp[-2].minor.yy242 = yylhsminor.yy242; break; - case 589: /* ordering_specification_opt ::= */ + case 590: /* ordering_specification_opt ::= */ { yymsp[1].minor.yy48 = ORDER_ASC; } break; - case 590: /* ordering_specification_opt ::= ASC */ + case 591: /* ordering_specification_opt ::= ASC */ { yymsp[0].minor.yy48 = ORDER_ASC; } break; - case 591: /* ordering_specification_opt ::= DESC */ + case 592: /* ordering_specification_opt ::= DESC */ { yymsp[0].minor.yy48 = ORDER_DESC; } break; - case 592: /* null_ordering_opt ::= */ + case 593: /* null_ordering_opt ::= */ { yymsp[1].minor.yy687 = NULL_ORDER_DEFAULT; } break; - case 593: /* null_ordering_opt ::= NULLS FIRST */ + case 594: /* null_ordering_opt ::= NULLS FIRST */ { yymsp[-1].minor.yy687 = NULL_ORDER_FIRST; } break; - case 594: /* null_ordering_opt ::= NULLS LAST */ + case 595: /* null_ordering_opt ::= NULLS LAST */ { yymsp[-1].minor.yy687 = NULL_ORDER_LAST; } break; default: From b83a4b14e7c73fcb0327eaf7412003445eefd686 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 29 May 2023 17:34:35 +0800 Subject: [PATCH 011/122] allow omit every clause for single point interp --- source/libs/parser/src/parTranslater.c | 33 ++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 39dab556f3..b2adbedd47 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3466,6 +3466,22 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { return TSDB_CODE_SUCCESS; } +static int32_t createDefaultEveryNode(STranslateContext* pCxt, SNode** pOutput) { + SValueNode* pEvery = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pEvery) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pEvery->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pEvery->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pEvery->isDuration = true; + pEvery->literal = taosStrdup("1s"); + + + *pOutput = (SNode*)pEvery; + return TSDB_CODE_SUCCESS; +} + static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { int32_t len = strlen(pInterval->literal); @@ -3481,7 +3497,12 @@ static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) { int32_t code = TSDB_CODE_SUCCESS; - code = checkEvery(pCxt, (SValueNode*)(*pEvery)); + if (NULL == *pEvery) { + code = createDefaultEveryNode(pCxt, pEvery); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkEvery(pCxt, (SValueNode*)(*pEvery)); + } if (TSDB_CODE_SUCCESS == code) { code = translateExpr(pCxt, pEvery); } @@ -3527,8 +3548,12 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { } if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, - "Missing RANGE clause, EVERY clause or FILL clause"); + if (QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) { + // single point interp every can be omitted + } else { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, + "Missing RANGE clause, EVERY clause or FILL clause"); + } } int32_t code = translateExpr(pCxt, &pSelect->pRange); @@ -3744,7 +3769,7 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = replaceTbName(pCxt, pSelect); } - + return code; } From 9501d4f4177373d826ec967e967829789e36f05e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 19:44:11 +0800 Subject: [PATCH 012/122] feat: allow user to reset root passwd --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 3 +++ source/dnode/mnode/impl/src/mndProfile.c | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 157e37f080..6a7e36fcbb 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -83,6 +83,7 @@ extern int64_t tsVndCommitMaxIntervalMs; // mnode extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndLogRetention; +extern bool tsMndSkipGrant; // monitor extern bool tsEnableMonitor; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c648f8551a..a43643a924 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -72,6 +72,7 @@ int64_t tsVndCommitMaxIntervalMs = 600 * 1000; // mnode int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; +bool tsMndSkipGrant = false; // monitor bool tsEnableMonitor = true; @@ -489,6 +490,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, 0) != 0) return -1; if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; @@ -887,6 +889,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; + tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 01dd223b5f..a1d815189c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -245,7 +245,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) { + if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0 && !tsMndSkipGrant) { mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd); code = TSDB_CODE_MND_AUTH_FAILURE; goto _OVER; From 267cbfbc1ce6f9721f90d03b131b426ae558d60c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 29 May 2023 19:49:28 +0800 Subject: [PATCH 013/122] enh: update syncNodePropose --- source/libs/sync/src/syncMain.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f64c2a9560..ccf7c3e4a4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -618,8 +618,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ return -1; } - // not restored, vnode enable - if (!pSyncNode->restoreFinish && pSyncNode->vgId != 1) { + if (!pSyncNode->restoreFinish) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; sNError(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); From 6679b8af7b5128d56962a7985f8e259adcc2e21b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 30 May 2023 10:55:56 +0800 Subject: [PATCH 014/122] fix: release action.pCont on error --- source/dnode/mnode/impl/src/mndTrans.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7822a596d4..44114bac03 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -263,6 +263,7 @@ static int32_t mndTransDecodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pAct _OVER: *offset = dataPos; + taosMemoryFreeClear(action.pCont); return ret; } @@ -279,7 +280,6 @@ static SSdbRow *mndTransDecode(SSdbRaw *pRaw) { int32_t undoActionNum = 0; int32_t commitActionNum = 0; int32_t dataPos = 0; - STransAction action = {0}; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; @@ -360,7 +360,6 @@ _OVER: mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); taosMemoryFreeClear(pRow); - taosMemoryFreeClear(action.pCont); return NULL; } From 1009c8096711b269f7d76bbe6fca11daf008c732 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 30 May 2023 14:21:32 +0800 Subject: [PATCH 015/122] fix bug --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b2adbedd47..fb269020de 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3548,7 +3548,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { } if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) { - if (QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) { + if (pSelect->pRange != NULL && QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) { // single point interp every can be omitted } else { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, From 0dc8baede48f7669ef42ed8a051a5f6a0f0d25d7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 30 May 2023 14:21:47 +0800 Subject: [PATCH 016/122] add test cases --- tests/system-test/2-query/interp.py | 1149 ++++++++++++++++++++++++++- 1 file changed, 1148 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 121d4dcff6..061cc38b51 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -32,6 +32,12 @@ class TDTestCase: ctbname3_null = "ctb3_null" stbname_null = "stb_null" + tbname_single = "tb_single" + ctbname1_single = "ctb1_single" + ctbname2_single = "ctb2_single" + ctbname3_single = "ctb3_single" + stbname_single = "stb_single" + tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") @@ -4020,7 +4026,7 @@ class TDTestCase: tdLog.printNoPrefix("======step 15: test interp pseudo columns") tdSql.error(f"select _irowts, c6 from {dbname}.{tbname}") - tdLog.printNoPrefix("======step 15: test interp in nested query") + tdLog.printNoPrefix("======step 16: test interp in nested query") tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{stbname}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") @@ -4038,6 +4044,1147 @@ class TDTestCase: tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select {ctbname1}.ts,{ctbname1}.c0 from {dbname}.{ctbname1}, {dbname}.{ctbname2} where {ctbname1}.ts = {ctbname2}.ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdLog.printNoPrefix("======step 17: test interp single point") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname_single} + (ts timestamp, c0 int) + ''' + ) + + tdSql.execute(f"insert into {dbname}.{tbname_single} values ('2020-02-01 00:00:01', 1)") + tdSql.execute(f"insert into {dbname}.{tbname_single} values ('2020-02-01 00:00:03', 3)") + tdSql.execute(f"insert into {dbname}.{tbname_single} values ('2020-02-01 00:00:05', 5)") + + tdSql.execute( + f'''create table if not exists {dbname}.{stbname_single} + (ts timestamp, c0 int, c1 float, c2 bool) tags (t0 int) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname1_single} using {dbname}.{stbname_single} tags(1) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname2_single} using {dbname}.{stbname_single} tags(2) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname3_single} using {dbname}.{stbname_single} tags(3) + ''' + ) + + tdSql.execute(f"insert into {dbname}.{ctbname1_single} values ('2020-02-01 00:00:01', 1, 1.0, true)") + + tdSql.execute(f"insert into {dbname}.{ctbname2_single} values ('2020-02-01 00:00:03', 3, 3.0, false)") + + tdSql.execute(f"insert into {dbname}.{ctbname3_single} values ('2020-02-01 00:00:05', 5, 5.0, true)") + + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(prev)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(next)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(linear)") + tdSql.checkRows(0) + + #super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(prev)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(next)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(linear)") + tdSql.checkRows(0) + + # partition by tbname + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(null)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(null)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(value,0)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(value,0)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(prev)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(prev)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, False) + tdSql.checkData(1, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03') fill(prev)") + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, False) + tdSql.checkData(1, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04') fill(prev)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 1, False) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05') fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 1, False) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(next)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03') fill(next)") + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03') fill(linear)") + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(linear)") + tdSql.checkRows(0) + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 83e9be2e40abf0ae1fbfbb25c48afb915eb1b631 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 30 May 2023 14:39:31 +0800 Subject: [PATCH 017/122] change zh docs --- docs/zh/12-taos-sql/06-select.md | 2 +- docs/zh/12-taos-sql/10-function.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 870df73471..5bc67755f0 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -55,7 +55,7 @@ window_clause: { | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] interp_clause: - RANGE(ts_val, ts_val) EVERY(every_val) FILL(fill_mod_and_val) + RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val) partition_by_clause: PARTITION BY expr [, expr] ... diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index b4785dc5e6..416d41614d 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -890,9 +890,10 @@ ignore_null_values: { - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 - INTERP 需要同时与 RANGE，EVERY 和 FILL 关键字一起使用。 -- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 +- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 - INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)，1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值. - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句) +- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如：SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear). - INTERP 作用于超级表时, 会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.2.0版本以后支持)。 - INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0版本以后支持)。 From 34fa89da0df5bb6fc90324f34177284d2a73306c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 30 May 2023 14:50:03 +0800 Subject: [PATCH 018/122] add en docs --- docs/en/12-taos-sql/06-select.md | 2 +- docs/en/12-taos-sql/10-function.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index ea0d7abc16..b28d5acb18 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -55,7 +55,7 @@ window_clause: { | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] interp_clause: - RANGE(ts_val, ts_val) EVERY(every_val) FILL(fill_mod_and_val) + RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val) partition_by_clause: PARTITION BY expr [, expr] ... diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index b469da8f55..b517bcb3cc 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -889,9 +889,10 @@ ignore_null_values: { - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter. - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input. - `INTERP` must be used along with `RANGE`, `EVERY`, `FILL` keywords. -- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. +- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range. timestamp2 is the ending point of the output time range. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY(time_unit)`. Starting from timestamp1, one interpolation is performed for every time interval specified `time_unit` parameter. The parameter `time_unit` must be an integer, with no quotes, with a time unit of: a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. For more information about FILL clause, see [FILL Clause](../distinguished/#fill-clause). +- When only one timestamp value is specified in `RANGE` clause, `INTERP` is used to generate interpolation at this point in time. In this case, `EVERY` clause can be omitted. For example, SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear). - `INTERP` can be applied to supertable by interpolating primary key sorted data of all its childtables. It can also be used with `partition by tbname` when applied to supertable to generate interpolation on each single timeline. - Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.2.0). - Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.3.0). @@ -902,7 +903,7 @@ ignore_null_values: { - We want to downsample every 1 hour and use a linear fill for missing values. Note the order in which the "partition by" clause and the "range", "every" and "fill" parameters are used. ```sql -SELECT _irowts,INTERP(current) FROM test.meters PARTITION BY TBNAME RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(LINEAR) +SELECT _irowts,INTERP(current) FROM test.meters PARTITION BY TBNAME RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(LINEAR) ``` ### LAST From 4a6d81855ba4ddefd15e96323ee7222c5fbe52b7 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 30 May 2023 18:50:00 +0800 Subject: [PATCH 019/122] enh: add mndTransValidate for prepare actions --- include/common/tmsgdef.h | 2 +- source/dnode/mnode/impl/inc/mndTrans.h | 5 + source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndSync.c | 205 ++++++++++++++++++------ source/dnode/mnode/impl/src/mndTrans.c | 18 +-- source/dnode/mnode/impl/src/mndVgroup.c | 3 +- 6 files changed, 176 insertions(+), 58 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1f2d597496..ee1888ff49 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -150,7 +150,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_TMQ_HB, "consumer-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_UNUSED2, "unused2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VG, "create-vg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_TIMER, "tmq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index ac078ce1a3..625546aa55 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -91,6 +91,11 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans); void mndTransRefresh(SMnode *pMnode, STrans *pTrans); int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname); +SSdbRaw *mndTransEncode(STrans *pTrans); +SSdbRow *mndTransDecode(SSdbRaw *pRaw); +void mndTransDropData(STrans *pTrans); + +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 286a624383..7c2f8b5b65 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -27,6 +27,7 @@ void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void mndSortVnodeGid(SVgObj *pVgroup); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 02925f9d8c..5be60c00a9 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,6 +17,7 @@ #include "mndSync.h" #include "mndCluster.h" #include "mndTrans.h" +#include "mndVgroup.h" static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { if (pMsg == NULL || pMsg->pCont == NULL) { @@ -73,76 +74,188 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } -int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - SMnode *pMnode = pFsm->data; +static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { + SSdbRow *pRow = NULL; + int32_t code = -1; + + if (pAction->msgType == TDMT_MND_CREATE_VG) { + pRow = mndVgroupActionDecode(pAction->pRaw); + if (pRow == NULL) goto _OUT; + + SVgObj *pVgroup = sdbGetRowObj(pRow); + if (pVgroup == NULL) goto _OUT; + + int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); + if (maxVgId > pVgroup->vgId) { + mError("trans:%d, failed to satisfy vgroup id %d of prepare action. maxVgId:%d", pTrans->id, pVgroup->vgId, + maxVgId); + goto _OUT; + } + } + + code = 0; +_OUT: + taosMemoryFreeClear(pRow); + return code; +} + +static int32_t mndTransValidatePrepareStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = -1; + int32_t action = 0; + + int32_t numOfActions = taosArrayGetSize(pTrans->prepareActions); + if (numOfActions == 0) { + code = 0; + goto _OUT; + } + + mInfo("trans:%d, validate %d prepare actions.", pTrans->id, numOfActions); + + for (action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); + + if (pAction->actionType != TRANS_ACTION_RAW) { + mError("trans:%d, prepare action:%d of unexpected type:%d", pTrans->id, action, pAction->actionType); + goto _OUT; + } + + code = mndTransValidatePrepareAction(pMnode, pTrans, pAction); + if (code != 0) { + mError("trans:%d, failed to validate prepare action: %d, numOfActions:%d", pTrans->id, action, numOfActions); + goto _OUT; + } + } + + code = 0; +_OUT: + return code; +} + +static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { + if (pTrans->stage == TRN_STAGE_PREPARE) { + return mndTransValidatePrepareStage(pMnode, pTrans); + } + return 0; +} + +static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { + STrans *pTrans = NULL; + int32_t code = -1; + + SSdbRow *pRow = mndTransDecode(pRaw); + if (pRow == NULL) goto _OUT; + + pTrans = sdbGetRowObj(pRow); + if (pTrans == NULL) goto _OUT; + + code = mndTransValidateImp(pMnode, pTrans); + +_OUT: + if (pTrans) mndTransDropData(pTrans); + if (pRow) taosMemoryFreeClear(pRow); + return code; +} + +int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; - + STrans *pTrans = NULL; + int32_t code = -1; int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); + + if (transId <= 0) { + mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); + terrno = TSDB_CODE_INVALID_MSG; + goto _OUT; + } + mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 " role:%s raw:%p sec:%d seq:%" PRId64, transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), pRaw, pMgmt->transSec, pMgmt->transSeq); - if (pMeta->code == 0) { - int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); - return 0; - } - sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); + code = mndTransValidate(pMnode, pRaw); + if (code != 0) { + mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); + goto _OUT; } - taosThreadMutexLock(&pMgmt->lock); - pMgmt->errCode = pMeta->code; - - if (transId <= 0) { - taosThreadMutexUnlock(&pMgmt->lock); - mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); - } else if (transId == pMgmt->transId) { - if (pMgmt->errCode != 0) { - mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); - } else { - mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq); - } - pMgmt->transId = 0; - pMgmt->transSec = 0; - pMgmt->transSeq = 0; - tsem_post(&pMgmt->syncSem); - taosThreadMutexUnlock(&pMgmt->lock); - } else { - taosThreadMutexUnlock(&pMgmt->lock); - STrans *pTrans = mndAcquireTrans(pMnode, transId); - if (pTrans != NULL) { - mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", - transId, pTrans->createdTime, pMgmt->transId); - mndTransRefresh(pMnode, pTrans); - mndReleaseTrans(pMnode, pTrans); - } else { - mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); - } + code = sdbWriteWithoutFree(pMnode->pSdb, pRaw); + if (code != 0) { + mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); + goto _OUT; } + pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans == NULL) { + mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); + goto _OUT; + } + + if (pTrans->stage == TRN_STAGE_PREPARE) { + bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans); + if (!continueExec) goto _OUT; + } + + if (pTrans->id != pMgmt->transId) { + mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", + pTrans->id, pTrans->createdTime, pMgmt->transId); + mndTransRefresh(pMnode, pTrans); + } + + sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta); + code = 0; + +_OUT: + if (pTrans) mndReleaseTrans(pMnode, pTrans); + return code; +} + +static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId == 0) { + goto _OUT; + } + + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + pMgmt->errCode = code; + tsem_post(&pMgmt->syncSem); + + if (pMgmt->errCode != 0) { + mError("trans:%d, failed to propose since %s, post sem", pMgmt->transId, tstrerror(pMgmt->errCode)); + } else { + mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, pMgmt->transId, pMgmt->transSeq); + } + +_OUT: + taosThreadMutexUnlock(&pMgmt->lock); return 0; } int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - int32_t code = 0; + SMnode *pMnode = pFsm->data; + int32_t code = pMsg->code; + if (code != 0) { + goto _OUT; + } + pMsg->info.conn.applyIndex = pMeta->index; pMsg->info.conn.applyTerm = pMeta->term; - if (pMsg->code == 0) { - SMnode *pMnode = pFsm->data; - atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex); - } + atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex); if (!syncUtilUserCommit(pMsg->msgType)) { - goto _out; + goto _OUT; } - code = mndProcessWriteMsg(pFsm, pMsg, pMeta); -_out: + code = mndProcessWriteMsg(pMnode, pMsg, pMeta); + +_OUT: + mndPostMgmtCode(pMnode, code); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; return code; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 44114bac03..14f12fab22 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -28,8 +28,6 @@ #define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 48 -static SSdbRaw *mndTransEncode(STrans *pTrans); -static SSdbRow *mndTransDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); @@ -38,14 +36,12 @@ static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); -static void mndTransDropData(STrans *pTrans); static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans); @@ -142,7 +138,7 @@ _OVER: return ret; } -static SSdbRaw *mndTransEncode(STrans *pTrans) { +SSdbRaw *mndTransEncode(STrans *pTrans) { terrno = TSDB_CODE_INVALID_MSG; int8_t sver = taosArrayGetSize(pTrans->prepareActions) ? TRANS_VER2_NUMBER : TRANS_VER1_NUMBER; @@ -267,7 +263,7 @@ _OVER: return ret; } -static SSdbRow *mndTransDecode(SSdbRaw *pRaw) { +SSdbRow *mndTransDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_INVALID_MSG; SSdbRow *pRow = NULL; @@ -444,7 +440,7 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { return 0; } -static void mndTransDropData(STrans *pTrans) { +void mndTransDropData(STrans *pTrans) { if (pTrans->prepareActions != NULL) { mndTransDropActions(pTrans->prepareActions); pTrans->prepareActions = NULL; @@ -1330,7 +1326,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) return code; } -static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = true; int32_t code = 0; @@ -1341,7 +1337,11 @@ static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); - mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction); + if (code != 0) { + mError("trans:%d, failed to execute prepare action:%d, numOfActions:%d", pTrans->id, action, numOfActions); + return false; + } } _OVER: diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 58612efe48..776c263edd 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -28,7 +28,6 @@ #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 -static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); @@ -1251,7 +1250,7 @@ int32_t mndAddPrepareNewVgAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVg) { SSdbRaw *pRaw = mndVgroupActionEncode(pVg); if (pRaw == NULL) goto _err; - STransAction action = {.pRaw = pRaw}; + STransAction action = {.pRaw = pRaw, .msgType = TDMT_MND_CREATE_VG}; if (mndTransAppendPrepareAction(pTrans, &action) != 0) goto _err; (void)sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); pRaw = NULL; From 28efd7dcd716fdc0e07acd74e305525d03dcab76 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 31 May 2023 10:53:36 +0800 Subject: [PATCH 020/122] enh: check trans conflicts in mndTransValidate --- include/libs/sync/sync.h | 10 +++++----- source/dnode/mnode/impl/src/mndSync.c | 16 +++++++++++++--- source/dnode/vnode/src/vnd/vnodeSync.c | 6 +++--- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e86a4f9690..2a0a4b0f63 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -154,14 +154,14 @@ typedef struct SSnapshotMeta { typedef struct SSyncFSM { void* data; - int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); SyncIndex (*FpAppliedIndexCb)(const struct SSyncFSM* pFsm); - int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); - void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); + void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm, const SyncIndex commitIdx); - void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); - void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SReConfigCbMeta* pMeta); + void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 5be60c00a9..3292d8fac2 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -133,6 +133,11 @@ _OUT: static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { if (pTrans->stage == TRN_STAGE_PREPARE) { + if (mndTransCheckConflict(pMnode, pTrans) < 0) { + mError("trans:%d, failed to validate trans conflicts.", pTrans->id); + return -1; + } + return mndTransValidatePrepareStage(pMnode, pTrans); } return 0; @@ -153,10 +158,12 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { _OUT: if (pTrans) mndTransDropData(pTrans); if (pRow) taosMemoryFreeClear(pRow); + if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); return code; } -int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { + terrno = TSDB_CODE_SUCCESS; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; STrans *pTrans = NULL; @@ -177,6 +184,8 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, const SFsmCbMeta *pMet code = mndTransValidate(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); + code = 0; + pMeta->code = terrno; goto _OUT; } @@ -236,7 +245,7 @@ _OUT: return 0; } -int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { SMnode *pMnode = pFsm->data; int32_t code = pMsg->code; if (code != 0) { @@ -245,6 +254,7 @@ int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta * pMsg->info.conn.applyIndex = pMeta->index; pMsg->info.conn.applyTerm = pMeta->term; + pMeta->code = 0; atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex); @@ -255,7 +265,7 @@ int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta * code = mndProcessWriteMsg(pMnode, pMsg, pMeta); _OUT: - mndPostMgmtCode(pMnode, code); + mndPostMgmtCode(pMnode, code ? code : pMeta->code); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 29f1ddc50f..ff551e6534 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -431,7 +431,7 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } -static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (pMsg->code == 0) { return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } @@ -451,7 +451,7 @@ static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFs return 0; } -static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (pMeta->isWeak == 1) { return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } @@ -463,7 +463,7 @@ static SyncIndex vnodeSyncAppliedIndex(const SSyncFSM *pFSM) { return atomic_load_64(&pVnode->state.applied); } -static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), From 60100a0fb6dd54f920bf8a5535b0cce01d80c9f3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 31 May 2023 15:34:32 +0800 Subject: [PATCH 021/122] feature: add user priv when create db, remove priv check with create_dbs --- source/dnode/mnode/impl/inc/mndUser.h | 2 + source/dnode/mnode/impl/src/mndDb.c | 22 ++++- source/dnode/mnode/impl/src/mndUser.c | 4 +- source/libs/catalog/src/ctgUtil.c | 3 + .../script/tsim/user/privilege_create_db.sim | 97 +++++++++++++++++++ tests/script/win-test-file | 1 + 6 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 tests/script/tsim/user/privilege_create_db.sim diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index aa7f97f087..93ae38e554 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -40,6 +40,8 @@ int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int3 int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); +void mndUserFreeObj(SUserObj *pUser); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index a451ae9df5..70214e80a5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -446,7 +446,8 @@ static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } -static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups, + SUserObj *pUserDuped) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); if (pDbRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; @@ -459,6 +460,13 @@ static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj * if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } + if (pUserDuped) { + SSdbRaw *pUserRaw = mndUserActionEncode(pUserDuped); + if (pUserRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pUserRaw) != 0) return -1; + if (sdbSetRawStatus(pUserRaw, SDB_STATUS_READY) != 0) return -1; + } + return 0; } @@ -565,6 +573,15 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, return -1; } + // add database privileges for user + SUserObj newUserObj = {0}, *pNewUserDuped = NULL; + if (!pUser->superUser) { + if (mndUserDupObj(pUser, &newUserObj) != 0) goto _OVER; + taosHashPut(newUserObj.readDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); + taosHashPut(newUserObj.writeDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); + pNewUserDuped = &newUserObj; + } + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-db"); if (pTrans == NULL) goto _OVER; @@ -577,7 +594,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mndTransSetOper(pTrans, MND_OPER_CREATE_DB); if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; - if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups, pNewUserDuped) != 0) goto _OVER; if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -586,6 +603,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, _OVER: taosMemoryFree(pVgroups); + mndUserFreeObj(&newUserObj); mndTransDrop(pTrans); return code; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 3da594109a..90d16a0a81 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -488,7 +488,7 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) { return pNew; } -static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; pNew->updateTime = taosGetTimestampMs(); @@ -508,7 +508,7 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { return 0; } -static void mndUserFreeObj(SUserObj *pUser) { +void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cf864e8643..6f4347a49c 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1555,10 +1555,13 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&pReq->tbName, dbFName); + // since that we add read/write previliges when create db, there is no need to check createdDbs +#if 0 if (pInfo->createdDbs && taosHashGet(pInfo->createdDbs, dbFName, strlen(dbFName))) { pRes->pass = true; return TSDB_CODE_SUCCESS; } +#endif switch (pReq->type) { case AUTH_TYPE_READ: { diff --git a/tests/script/tsim/user/privilege_create_db.sim b/tests/script/tsim/user/privilege_create_db.sim new file mode 100644 index 0000000000..c81bd1b258 --- /dev/null +++ b/tests/script/tsim/user/privilege_create_db.sim @@ -0,0 +1,97 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========================root user create user +sql create user u1 pass "taosdata" +sql create user u2 pass "taosdata" +sql create database test +sql select * from information_schema.ins_user_privileges where user_name == "root" +if $rows != 1 then + return -1 +endi + +print =============connect with u1 +sql connect u1 +sql create database u1_d1 +sql use u1_d1 +sql create table u1_d1.t1(ts timestamp, c2 int) +sql use information_schema +sql select * from ins_user_privileges where user_name == "u1" order by privilege +if $rows != 2 then + return -1 +endi +if $data01 != read then + return -1 +endi +if $data11 != write then + return -1 +endi +if $data02 != u1_d1 then + return -1 +endi +if $data12 != u1_d1 then + return -1 +endi + +sql_error grant all on *.* to u1 +sql_error grant all on test.* to u1 + +print =============connect with u2 +sql connect u2 +sql create database u2_d1 +sql use u2_d1 +sql create table u2_d1.t1(ts timestamp, c2 int) +sql use information_schema +sql select * from ins_user_privileges where user_name == "u2" order by privilege +if $rows != 2 then + return -1 +endi +if $data01 != read then + return -1 +endi +if $data11 != write then + return -1 +endi +if $data02 != u2_d1 then + return -1 +endi +if $data12 != u2_d1 then + return -1 +endi + +sql_error select * from u1_d1.t1 +sql_error revoke read on u2_d1.* from u2 + +print =============connect with root, revoke read from u1, all from u2 +sql connect +sql revoke read on u1_d1.* from u1 +sql revoke all on u2_d1.* from u2 +sleep 1000 + +print =============connect with u1 +sql connect u1 +sql insert into u1_d1.t1 values(now, 1) +sql_error select * from u1_d1.t1; + +print =============connect with u2 +sql connect u2 +sql_error select * from u2_d1.t1; +sql_error insert into u2_d1.t1 values(now, 1) + +print =============connect with root, grant read to u1, all to u2 +sql connect +sql grant read on u1_d1.* to u1 +sql grant all on u2_d1.* to u2 + +sleep 1000 +print =============connect with u1 +sql connect u1 +sql select * from u1_d1.t1; +sql insert into u1_d1.t1 values(now, 2) + +print =============connect with u2 +sql connect u2 +sql select * from u2_d1.t1; +sql insert into u2_d1.t1 values(now, 2) diff --git a/tests/script/win-test-file b/tests/script/win-test-file index b7fbbed5c1..adef71cb45 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -4,6 +4,7 @@ ./test.sh -f tsim/user/privilege_sysinfo.sim ./test.sh -f tsim/user/privilege_topic.sim ./test.sh -f tsim/user/privilege_table.sim +./test.sh -f tsim/user/privilege_create_db.sim ./test.sh -f tsim/db/alter_option.sim rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/db/alter_replica_31.sim From fb2f367aacd17f9128996097e4b074f37dfe2199 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Jun 2023 08:13:32 +0800 Subject: [PATCH 022/122] meta/split: trim invalidated uids --- source/dnode/vnode/src/inc/vnodeInt.h | 16 +++---- source/dnode/vnode/src/meta/metaTable.c | 55 ++++++++++++++++++++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 17 ++++---- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index db285dc124..0bfdf33019 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -107,10 +107,10 @@ struct SQueryNode { typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; -SMTbCursor *metaOpenTbCursor(void *pVnode); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); +SMTbCursor* metaOpenTbCursor(void* pVnode); +void metaCloseTbCursor(SMTbCursor* pTbCur); +int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); #endif @@ -146,6 +146,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); +int32_t metaTrimTables(SMeta* pMeta); int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); @@ -154,8 +155,8 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); -int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); @@ -175,7 +176,7 @@ void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -491,7 +492,6 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4f9f0c688e..20cf423182 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -848,7 +848,52 @@ static void metaDropTables(SMeta *pMeta, SArray *tbUids) { metaULock(pMeta); } -int metaTrimTables(SMeta *pMeta, int64_t version) { +static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { + int32_t code = 0; + // 1, tranverse table's + // 2, validate table name using vnodeValidateTableHash + // 3, push invalidated table's uid into uidList + + TBC *pCur; + code = tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + if (code < 0) { + return code; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbTbcClose(pCur); + return code; + } + + void *pData = NULL; + int nData = 0; + + while (1) { + int32_t ret = tdbTbcNext(pCur, NULL, NULL, &pData, &nData); + if (ret < 0) { + break; + } + + SMetaEntry me = {0}; + SDecoder dc = {0}; + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &me); + if (me.type == TSDB_CHILD_TABLE) { + int32_t ret = vnodeValidateTableHash(pMeta->pVnode, me.name); + if (TSDB_CODE_VND_HASH_MISMATCH == ret) { + taosArrayPush(uidList, &me.uid); + } + } + tDecoderClear(&dc); + } + tdbFree(pData); + tdbTbcClose(pCur); + + return 0; +} + +int32_t metaTrimTables(SMeta *pMeta) { int32_t code = 0; SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); @@ -856,7 +901,7 @@ int metaTrimTables(SMeta *pMeta, int64_t version) { return TSDB_CODE_OUT_OF_MEMORY; } - // code = metaFilterTableByHash(pMeta, /*ttl, */ tbUids); + code = metaFilterTableByHash(pMeta, tbUids); if (code != 0) { goto end; } @@ -1027,7 +1072,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1); metaUidCacheClear(pMeta, e.ctbEntry.suid); - metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) @@ -1039,7 +1084,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaStatsCacheDrop(pMeta, uid); metaUidCacheClear(pMeta, uid); - metaTbGroupCacheClear(pMeta, uid); + metaTbGroupCacheClear(pMeta, uid); --pMeta->pVnode->config.vndStats.numOfSTables; } @@ -1460,7 +1505,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid); - metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); metaULock(pMeta); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b950437f23..c3fb5e5ad4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -245,11 +245,11 @@ _exit: static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; - int32_t size; - int32_t ret; - uint8_t *pCont; - SEncoder *pCoder = &(SEncoder){0}; - SDeleteRes res = {0}; + int32_t size; + int32_t ret; + uint8_t *pCont; + SEncoder *pCoder = &(SEncoder){0}; + SDeleteRes res = {0}; SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; initStorageAPI(&handle.api); @@ -316,8 +316,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg return -1; } - vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - ver); + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), ver); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); ASSERT(pVnode->state.applied + 1 == ver); @@ -1479,6 +1478,7 @@ static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] + code = metaTrimTables(pVnode->pMeta); return code; } @@ -1492,8 +1492,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pR code = vnodeConsolidateAlterHashRange(pVnode, ver); if (code < 0) { - vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), - ver); + vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), ver); goto _exit; } pVnode->config.hashChange = false; From 9b9bf2de6897a1453dfc31b9a13e77125a606d09 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Jun 2023 10:41:01 +0800 Subject: [PATCH 023/122] meta/tbc: key cannot be null --- source/dnode/vnode/src/meta/metaTable.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 20cf423182..eb169fbdc2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -866,11 +866,11 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { return code; } - void *pData = NULL; - int nData = 0; + void *pData = NULL, *pKey = NULL; + int nData = 0, nKey = 0; while (1) { - int32_t ret = tdbTbcNext(pCur, NULL, NULL, &pData, &nData); + int32_t ret = tdbTbcNext(pCur, &pKey, &nKey, &pData, &nData); if (ret < 0) { break; } @@ -879,7 +879,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { SDecoder dc = {0}; tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &me); - if (me.type == TSDB_CHILD_TABLE) { + if (me.type != TSDB_SUPER_TABLE) { int32_t ret = vnodeValidateTableHash(pMeta->pVnode, me.name); if (TSDB_CODE_VND_HASH_MISMATCH == ret) { taosArrayPush(uidList, &me.uid); @@ -888,6 +888,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { tDecoderClear(&dc); } tdbFree(pData); + tdbFree(pKey); tdbTbcClose(pCur); return 0; From 2ef2715ee13f6d4804f65b683871f237729640de Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 1 Jun 2023 17:58:27 +0800 Subject: [PATCH 024/122] enh: add mndAddVgStatusAction and mndAddDbStatusAction --- source/dnode/mnode/impl/src/mndVgroup.c | 67 ++++++++++++++++--------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 776c263edd..1eea005d2c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2250,10 +2250,13 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, return 0; } -static int32_t mndTransCommitVgStatus(STrans *pTrans, SVgObj *pVg, ESdbStatus vgStatus) { +typedef int32_t (*FpTransActionCb)(STrans *pTrans, SSdbRaw *pRaw); + +static int32_t mndAddVgStatusAction(STrans *pTrans, SVgObj *pVg, ESdbStatus vgStatus, ETrnStage stage) { + FpTransActionCb appendActionCb = (stage == TRN_STAGE_COMMIT_ACTION) ? mndTransAppendCommitlog : mndTransAppendRedolog; SSdbRaw *pRaw = mndVgroupActionEncode(pVg); if (pRaw == NULL) goto _err; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _err; + if (appendActionCb(pTrans, pRaw) != 0) goto _err; (void)sdbSetRawStatus(pRaw, vgStatus); pRaw = NULL; return 0; @@ -2262,10 +2265,22 @@ _err: return -1; } +static int32_t mndAddDbStatusAction(STrans *pTrans, SDbObj *pDb, ESdbStatus dbStatus, ETrnStage stage) { + FpTransActionCb appendActionCb = (stage == TRN_STAGE_COMMIT_ACTION) ? mndTransAppendCommitlog : mndTransAppendRedolog; + SSdbRaw *pRaw = mndDbActionEncode(pDb); + if (pRaw == NULL) goto _err; + if (appendActionCb(pTrans, pRaw) != 0) goto _err; + (void)sdbSetRawStatus(pRaw, dbStatus); + pRaw = NULL; + return 0; +_err: + sdbFreeRaw(pRaw); + return -1; +} + int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { int32_t code = -1; STrans *pTrans = NULL; - SSdbRaw *pRaw = NULL; SDbObj dbObj = {0}; SArray *pArray = mndBuildDnodesArray(pMnode, 0); @@ -2339,24 +2354,13 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg2) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; - // adjust vgroup replica - if (pDb->cfg.replications != newVg1.replica) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER; - } else { - if (mndTransCommitVgStatus(pTrans, &newVg1, SDB_STATUS_READY) < 0) goto _OVER; - } - - if (pDb->cfg.replications != newVg2.replica) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER; - } else { - if (mndTransCommitVgStatus(pTrans, &newVg2, SDB_STATUS_READY) < 0) goto _OVER; - } - - if (mndTransCommitVgStatus(pTrans, pVgroup, SDB_STATUS_DROPPED) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + // update db status memcpy(&dbObj, pDb, sizeof(SDbObj)); if (dbObj.cfg.pRetensions != NULL) { dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); @@ -2365,11 +2369,27 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro dbObj.vgVersion++; dbObj.updateTime = taosGetTimestampMs(); dbObj.cfg.numOfVgroups++; - pRaw = mndDbActionEncode(&dbObj); - if (pRaw == NULL) goto _OVER; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + if (mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + + // adjust vgroup replica + if (pDb->cfg.replications != newVg1.replica) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER; + } else { + if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + } + + if (pDb->cfg.replications != newVg2.replica) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER; + } else { + if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + } + + if (mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; + + // commit db status + dbObj.vgVersion++; + dbObj.updateTime = taosGetTimestampMs(); + if (mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -2377,7 +2397,6 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro _OVER: taosArrayDestroy(pArray); mndTransDrop(pTrans); - sdbFreeRaw(pRaw); taosArrayDestroy(dbObj.cfg.pRetensions); return code; } From 8c00af5c1715ed81d4559e733c5c2f42e3aad64f Mon Sep 17 00:00:00 2001 From: huolibo Date: Thu, 1 Jun 2023 18:19:17 +0800 Subject: [PATCH 025/122] other: add jdbc consumer demo --- examples/JDBC/consumer-demo/pom.xml | 70 +++++++++++++++++ examples/JDBC/consumer-demo/readme.md | 52 +++++++++++++ .../src/main/java/com/taosdata/Bean.java | 43 ++++++++++ .../java/com/taosdata/BeanDeserializer.java | 6 ++ .../src/main/java/com/taosdata/Config.java | 78 +++++++++++++++++++ .../main/java/com/taosdata/ConsumerDemo.java | 65 ++++++++++++++++ .../src/main/java/com/taosdata/Worker.java | 60 ++++++++++++++ 7 files changed, 374 insertions(+) create mode 100644 examples/JDBC/consumer-demo/pom.xml create mode 100644 examples/JDBC/consumer-demo/readme.md create mode 100644 examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java create mode 100644 examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java create mode 100644 examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java create mode 100644 examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java create mode 100644 examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java diff --git a/examples/JDBC/consumer-demo/pom.xml b/examples/JDBC/consumer-demo/pom.xml new file mode 100644 index 0000000000..aa3cb154e5 --- /dev/null +++ b/examples/JDBC/consumer-demo/pom.xml @@ -0,0 +1,70 @@ + + + 4.0.0 + + com.taosdata + consumer + 1.0-SNAPSHOT + + + 8 + 8 + + + + + com.taosdata.jdbc + taos-jdbcdriver + 3.2.1 + + + com.google.guava + guava + 30.1.1-jre + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + ConsumerDemo + + ConsumerDemo + + + com.taosdata.ConsumerDemo + + + + jar-with-dependencies + + + package + + single + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + UTF-8 + + + + + + + \ No newline at end of file diff --git a/examples/JDBC/consumer-demo/readme.md b/examples/JDBC/consumer-demo/readme.md new file mode 100644 index 0000000000..77742ab605 --- /dev/null +++ b/examples/JDBC/consumer-demo/readme.md @@ -0,0 +1,52 @@ +# How to Run the Consumer Demo Code On Linux OS +TDengine's Consumer demo project is organized in a Maven way so that users can easily compile, package and run the project. If you don't have Maven on your server, you may install it using +``` +sudo apt-get install maven +``` + +## Install TDengine Client +Make sure you have already installed a tdengine client on your current develop environment. +Download the tdengine package on our website: ``https://www.taosdata.com/cn/all-downloads/`` and install the client. + +## Run Consumer Demo using mvn plugin +run command: +``` +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.ConsumerDemo" +``` + +## Custom configuration +```shell +# the host of TDengine server +export TAOS_HOST="127.0.0.1" + +# the port of TDengine server +export TAOS_PORT="6041" + +# the consumer type, can be "ws" or "jni" +export TAOS_TYPE="ws" + +# the number of consumers +export TAOS_JDBC_CONSUMER_NUM="1" + +# the number of processors to consume +export TAOS_JDBC_PROCESSOR_NUM="2" + +# the number of records to be consumed per processor per second +export TAOS_JDBC_RATE_PER_PROCESSOR="1000" + +# poll wait time in ms +export TAOS_JDBC_POLL_SLEEP="100" +``` + +## Run Consumer Demo using jar + +To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/consumer-demo`` and execute +``` +mvn clean package assembly:single +``` + +To run ConsumerDemo.jar, go to ``TDengine/tests/examples/JDBC/consumer-demo`` and execute +``` +java -jar target/ConsumerDemo-jar-with-dependencies.jar +``` + diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java new file mode 100644 index 0000000000..2f2467b371 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java @@ -0,0 +1,43 @@ +package com.taosdata; + +import java.sql.Timestamp; + +public class Bean { + private Timestamp ts; + private Integer c1; + private String c2; + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public Integer getC1() { + return c1; + } + + public void setC1(Integer c1) { + this.c1 = c1; + } + + public String getC2() { + return c2; + } + + public void setC2(String c2) { + this.c2 = c2; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Bean {"); + sb.append("ts=").append(ts); + sb.append(", c1=").append(c1); + sb.append(", c2='").append(c2).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java new file mode 100644 index 0000000000..478af9e70d --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java @@ -0,0 +1,6 @@ +package com.taosdata; + +import com.taosdata.jdbc.tmq.ReferenceDeserializer; + +public class BeanDeserializer extends ReferenceDeserializer { +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java new file mode 100644 index 0000000000..08579926e3 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java @@ -0,0 +1,78 @@ +package com.taosdata; + +public class Config { + public static final String TOPIC = "test_consumer"; + public static final String TAOS_HOST = "127.0.0.1"; + public static final String TAOS_PORT = "6041"; + public static final String TAOS_TYPE = "ws"; + public static final int TAOS_JDBC_CONSUMER_NUM = 1; + public static final int TAOS_JDBC_PROCESSOR_NUM = 2; + public static final int TAOS_JDBC_RATE_PER_PROCESSOR = 1000; + public static final int TAOS_JDBC_POLL_SLEEP = 100; + + private final int consumerNum; + private final int processCapacity; + private final int rate; + private final int pollSleep; + private final String type; + private final String host; + private final String port; + + public Config(String type, String host, String port, int consumerNum, int processCapacity, int rate, int pollSleep) { + this.type = type; + this.consumerNum = consumerNum; + this.processCapacity = processCapacity; + this.rate = rate; + this.pollSleep = pollSleep; + this.host = host; + this.port = port; + } + + public int getConsumerNum() { + return consumerNum; + } + + public int getProcessCapacity() { + return processCapacity; + } + + public int getRate() { + return rate; + } + + public int getPollSleep() { + return pollSleep; + } + + public String getHost() { + return host; + } + + public String getPort() { + return port; + } + + public String getType() { + return type; + } + + public static Config getFromENV() { + String host = System.getenv("TAOS_HOST") != null ? System.getenv("TAOS_HOST") : TAOS_HOST; + String port = System.getenv("TAOS_PORT") != null ? System.getenv("TAOS_PORT") : TAOS_PORT; + String type = System.getenv("TAOS_TYPE") != null ? System.getenv("TAOS_TYPE") : TAOS_TYPE; + + String c = System.getenv("TAOS_JDBC_CONSUMER_NUM"); + int num = c != null ? Integer.parseInt(c) : TAOS_JDBC_CONSUMER_NUM; + + String p = System.getenv("TAOS_JDBC_PROCESSOR_NUM"); + int capacity = p != null ? Integer.parseInt(p) : TAOS_JDBC_PROCESSOR_NUM; + + String r = System.getenv("TAOS_JDBC_RATE_PER_PROCESSOR"); + int rate = r != null ? Integer.parseInt(r) : TAOS_JDBC_RATE_PER_PROCESSOR; + + String s = System.getenv("TAOS_JDBC_POLL_SLEEP"); + int sleep = s != null ? Integer.parseInt(s) : TAOS_JDBC_POLL_SLEEP; + + return new Config(type, host, port, num, capacity, rate, sleep); + } +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java new file mode 100644 index 0000000000..7c7719c639 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java @@ -0,0 +1,65 @@ +package com.taosdata; + +import com.taosdata.jdbc.tmq.TMQConstants; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.taosdata.Config.*; + +public class ConsumerDemo { + public static void main(String[] args) throws SQLException { + // Config + Config config = Config.getFromENV(); + // Generated data + mockData(); + + Properties prop = new Properties(); + prop.setProperty(TMQConstants.CONNECT_TYPE, config.getType()); + prop.setProperty(TMQConstants.BOOTSTRAP_SERVERS, config.getHost() + ":" + config.getPort()); + prop.setProperty(TMQConstants.CONNECT_USER, "root"); + prop.setProperty(TMQConstants.CONNECT_PASS, "taosdata"); + prop.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); + prop.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); + prop.setProperty(TMQConstants.GROUP_ID, "gId"); + prop.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taosdata.BeanDeserializer"); + for (int i = 0; i < config.getConsumerNum() - 1; i++) { + new Thread(new Worker(prop, config)).start(); + } + new Worker(prop, config).run(); + } + + public static void mockData() throws SQLException { + String dbName = "test_consumer"; + String tableName = "st"; + String url = "jdbc:TAOS-RS://" + TAOS_HOST + ":" + TAOS_PORT + "/?user=root&password=taosdata&batchfetch=true"; + Connection connection = DriverManager.getConnection(url); + Statement statement = connection.createStatement(); + statement.executeUpdate("create database if not exists " + dbName + " WAL_RETENTION_PERIOD 3650"); + statement.executeUpdate("use " + dbName); + statement.executeUpdate("create table if not exists " + tableName + " (ts timestamp, c1 int, c2 nchar(100)) "); + statement.executeUpdate("create topic if not exists " + TOPIC + " as select ts, c1, c2 from " + tableName); + + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r); + t.setName("mock-data-thread-" + t.getId()); + return t; + }); + AtomicInteger atomic = new AtomicInteger(); + scheduledExecutorService.scheduleWithFixedDelay(() -> { + int i = atomic.getAndIncrement(); + try { + statement.executeUpdate("insert into " + tableName + " values(now, " + i + ",'" + i + "')"); + } catch (SQLException e) { + // ignore + } + }, 0, 10, TimeUnit.MILLISECONDS); + } +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java new file mode 100644 index 0000000000..f6e21cd729 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java @@ -0,0 +1,60 @@ +package com.taosdata; + +import com.google.common.util.concurrent.RateLimiter; +import com.taosdata.jdbc.tmq.ConsumerRecord; +import com.taosdata.jdbc.tmq.ConsumerRecords; +import com.taosdata.jdbc.tmq.TaosConsumer; + +import java.sql.SQLException; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; + +public class Worker implements Runnable { + + int sleepTime; + int rate; + + ForkJoinPool pool = new ForkJoinPool(); + Semaphore semaphore; + + TaosConsumer consumer; + + public Worker(Properties prop, Config config) throws SQLException { + consumer = new TaosConsumer<>(prop); + consumer.subscribe(Collections.singletonList(Config.TOPIC)); + semaphore = new Semaphore(config.getProcessCapacity()); + sleepTime = config.getPollSleep(); + rate = config.getRate(); + } + + @Override + public void run() { + while (!Thread.interrupted()) { + try { + // 控制请求频率 + if (semaphore.tryAcquire()) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(sleepTime)); + pool.submit(() -> { + RateLimiter limiter = RateLimiter.create(rate); + try { + for (ConsumerRecord record : records) { + // 流量控制 + limiter.acquire(); + // 业务处理数据 + System.out.println("[" + LocalDateTime.now() + "] Thread id:" + Thread.currentThread().getId() + " -> " + record.value()); + } + } finally { + semaphore.release(); + } + }); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + } +} From 738cfd5d8d1914b590d7d94f04c41132d6f27a50 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 23:14:30 +0800 Subject: [PATCH 026/122] fix:open compatibility test cases --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6a9a2b609a..0b235d0dc8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -564,7 +564,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py -#,,n,system-test,python3 ./test.py -f 0-others/compatibility.py +,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py From 6dda2e94f403e87888c77f095bf87fedb3743b2e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Jun 2023 11:42:25 +0800 Subject: [PATCH 027/122] fix: distinct primary timestamp issue --- source/libs/parser/src/parAstCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index b707f217e3..a4068930fe 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -828,7 +828,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->pProjectionList = pProjectionList; select->pFromTable = pTable; sprintf(select->stmtName, "%p", select); - select->timeLineResMode = TIME_LINE_GLOBAL; + select->timeLineResMode = select->isDistinct ? TIME_LINE_NONE : TIME_LINE_GLOBAL; select->onlyHasKeepOrderFunc = true; select->timeRange = TSWINDOW_INITIALIZER; return (SNode*)select; From ae26f701ae12f8b177c95e9370911ceb6db26c5b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 2 Jun 2023 18:41:30 +0800 Subject: [PATCH 028/122] fix: ignore TSDB_CODE_MND_TRANS_SYNC_TIMEOUT on updating trans --- include/util/taoserror.h | 2 +- source/dnode/mnode/impl/src/mndSync.c | 2 ++ source/dnode/mnode/impl/src/mndTrans.c | 4 ++-- source/util/src/terror.c | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636a5e63a7..40b84f8e97 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -345,7 +345,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D4) #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) #define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal -#define TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) +#define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) // mnode-mq diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3292d8fac2..68bfe09b5e 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -192,6 +192,8 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { code = sdbWriteWithoutFree(pMnode->pSdb, pRaw); if (code != 0) { mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); + code = 0; + pMeta->code = terrno; goto _OUT; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 14f12fab22..7ebaf6dda5 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -511,7 +511,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage), pNew->createdTime); // only occured while sync timeout - terrno = TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT; + terrno = TSDB_CODE_MND_TRANS_SYNC_TIMEOUT; return -1; } @@ -937,7 +937,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (code == TSDB_CODE_SYN_TIMEOUT) { - code = TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT; + code = TSDB_CODE_MND_TRANS_SYNC_TIMEOUT; } if (i != 0 && code == 0) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 31727f7535..febeae8181 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -275,7 +275,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog is null") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq From 532e02fb2adf4aa3e8e4ae5bd7b86c3b0c221dbd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 5 Jun 2023 10:47:15 +0800 Subject: [PATCH 029/122] enh: improve filter performance --- include/common/tdatablock.h | 1 + source/common/src/tdatablock.c | 23 +++++++++++++++++++++++ source/libs/executor/src/executorInt.c | 12 +----------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 33c571fc1b..53fc07c3f3 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -178,6 +178,7 @@ int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); +int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 311c79381c..a6a54a8347 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -126,6 +126,29 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const return 0; } +int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData) { + int32_t type = pColumnInfoData->info.type; + if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = 0; + if (type == TSDB_DATA_TYPE_JSON) { + dataLen = getJsonValueLen(pData); + } else { + dataLen = varDataTLen(pData); + } + + SVarColAttr* pAttr = &pColumnInfoData->varmeta; + + uint32_t len = pColumnInfoData->varmeta.length; + pColumnInfoData->varmeta.offset[dstRowIdx] = pColumnInfoData->varmeta.offset[srcRowIdx]; + } else { + memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * dstRowIdx, pData, pColumnInfoData->info.bytes); + colDataClearNull_f(pColumnInfoData->nullbitmap, dstRowIdx); + } + + return 0; +} + + int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 4f1a0254e4..ce7a117543 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -573,18 +573,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (colDataIsNull_var(pDst, j)) { colDataSetNull_var(pDst, numOfRows); } else { - // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - int32_t len = 0; - if (pDst->info.type == TSDB_DATA_TYPE_JSON) { - len = getJsonValueLen(p1); - } else { - len = varDataTLen(p1); - } - char* p2 = taosMemoryMalloc(len); - memcpy(p2, p1, len); - colDataSetVal(pDst, numOfRows, p2, false); - taosMemoryFree(p2); + colDataReassignVal(pDst, numOfRows, j, p1); } numOfRows += 1; j += 1; From 8aa2e96d4ed9bc8c2837bfb10340f7431f573f7e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 5 Jun 2023 11:26:45 +0800 Subject: [PATCH 030/122] fix: test case issue --- tests/system-test/2-query/interp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 121d4dcff6..a29a521574 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -4028,13 +4028,13 @@ class TDTestCase: tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1}) partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union select * from {dbname}.{ctbname2}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") - tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") - tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") - tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select {ctbname1}.ts,{ctbname1}.c0 from {dbname}.{ctbname1}, {dbname}.{ctbname2} where {ctbname1}.ts = {ctbname2}.ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") From 3e524d9aa363410ddb913c2007dd3a92e4591ea0 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Mon, 5 Jun 2023 15:02:00 +0800 Subject: [PATCH 031/122] docs: add assignments and seek offset --- docs/en/14-reference/03-connector/06-rust.mdx | 14 +++++++++++++- docs/zh/08-connector/26-rust.mdx | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 99c3d2c066..6af91d01ac 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -499,6 +499,18 @@ The TMQ is of [futures::Stream](https://docs.rs/futures/latest/futures/stream/in } ``` +Get assignments： + +```rust +let assignments = consumer.assignments().await.unwrap(); +``` + +Seek offset： + +```rust +consumer.offset_seek(topic, vgroup_id, offset).await; +``` + Unsubscribe: ```rust @@ -513,7 +525,7 @@ The following parameters can be configured for the TMQ DSN. Only `group.id` is m - `enable.auto.commit`: Automatically commits. This can be enabled when data consistency is not essential. - `auto.commit.interval.ms`: Interval for automatic commits. -For more information, see [GitHub sample file](https://github.com/taosdata/taos-connector-rust/blob/main/examples/subscribe.rs). +For more information, see [GitHub sample file](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/rust/nativeexample/examples/subscribe_demo.rs). For information about other structure APIs, see the [Rust documentation](https://docs.rs/taos). diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 41a429b026..095271e7da 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -502,6 +502,18 @@ TMQ 消息队列是一个 [futures::Stream](https://docs.rs/futures/latest/futur } ``` +获取消费进度： + +```rust +let assignments = consumer.assignments().await.unwrap(); +``` + +按照指定的进度消费： + +```rust +consumer.offset_seek(topic, vgroup_id, offset).await; +``` + 停止订阅： ```rust @@ -516,7 +528,7 @@ consumer.unsubscribe().await; - `enable.auto.commit`: 当设置为 `true` 时，将启用自动标记模式，当对数据一致性不敏感时，可以启用此方式。 - `auto.commit.interval.ms`: 自动标记的时间间隔。 -完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/taos-connector-rust/blob/main/examples/subscribe.rs). +完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/rust/nativeexample/examples/subscribe_demo.rs). 其他相关结构体 API 使用说明请移步 Rust 文档托管网页：。 From b1800a2bd5e6dd625d0328062abf5e86c247c065 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Mon, 5 Jun 2023 15:12:58 +0800 Subject: [PATCH 032/122] docs: add version requirements --- docs/zh/08-connector/26-rust.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 095271e7da..9bb444d9e8 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -504,12 +504,17 @@ TMQ 消息队列是一个 [futures::Stream](https://docs.rs/futures/latest/futur 获取消费进度： +版本要求 connector-rust >= v0.8.8， TDengine >= 3.0.5.0 + ```rust let assignments = consumer.assignments().await.unwrap(); ``` 按照指定的进度消费： +版本要求 connector-rust >= v0.8.8， TDengine >= 3.0.5.0 + + ```rust consumer.offset_seek(topic, vgroup_id, offset).await; ``` From 985362d94c3370e3a56c0a3cd5e3581814f16e60 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Mon, 5 Jun 2023 15:16:03 +0800 Subject: [PATCH 033/122] docs: add version --- docs/en/14-reference/03-connector/06-rust.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 6af91d01ac..650d0d516d 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -501,12 +501,16 @@ The TMQ is of [futures::Stream](https://docs.rs/futures/latest/futures/stream/in Get assignments： +Version requirements connector-rust >= v0.8.8， TDengine >= 3.0.5.0 + ```rust let assignments = consumer.assignments().await.unwrap(); ``` Seek offset： +Version requirements connector-rust >= v0.8.8， TDengine >= 3.0.5.0 + ```rust consumer.offset_seek(topic, vgroup_id, offset).await; ``` From 8ccfb2907997171aff471568477f24cea4be136d Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Mon, 5 Jun 2023 15:16:13 +0800 Subject: [PATCH 034/122] docs: fmt lines --- docs/zh/08-connector/26-rust.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 9bb444d9e8..e1e94e068f 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -514,7 +514,6 @@ let assignments = consumer.assignments().await.unwrap(); 版本要求 connector-rust >= v0.8.8， TDengine >= 3.0.5.0 - ```rust consumer.offset_seek(topic, vgroup_id, offset).await; ``` From 3d0b02f5d637f7ab54951ce2c195fb44ec30c04b Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Mon, 5 Jun 2023 15:17:07 +0800 Subject: [PATCH 035/122] docs: fmt lines --- docs/en/14-reference/03-connector/06-rust.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 650d0d516d..fb7c368df1 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -501,7 +501,7 @@ The TMQ is of [futures::Stream](https://docs.rs/futures/latest/futures/stream/in Get assignments： -Version requirements connector-rust >= v0.8.8， TDengine >= 3.0.5.0 +Version requirements connector-rust >= v0.8.8, TDengine >= 3.0.5.0 ```rust let assignments = consumer.assignments().await.unwrap(); @@ -509,7 +509,7 @@ let assignments = consumer.assignments().await.unwrap(); Seek offset： -Version requirements connector-rust >= v0.8.8， TDengine >= 3.0.5.0 +Version requirements connector-rust >= v0.8.8, TDengine >= 3.0.5.0 ```rust consumer.offset_seek(topic, vgroup_id, offset).await; From be34546edd8935917f05e663dede14a0b919e5af Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 5 Jun 2023 15:19:06 +0800 Subject: [PATCH 036/122] enh: enable interp fill value support scarlar expression --- source/libs/parser/src/parTranslater.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c10ee5d988..57f708c5cd 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -707,6 +707,10 @@ static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isInterpFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isInterpPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } @@ -3006,7 +3010,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { } static EDealRes needFillImpl(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { + if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { *(bool*)pContext = true; return DEAL_RES_END; } @@ -3517,6 +3521,9 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true); } + if (TSDB_CODE_SUCCESS == code) { + code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList); + } return code; } From d02746d86f3773664e990490e767ed2b717ed8e1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 5 Jun 2023 15:24:39 +0800 Subject: [PATCH 037/122] fix(query/sys): pause/resume with sys scan operator --- include/libs/executor/storageapi.h | 218 +++++++++++---------- source/dnode/vnode/src/inc/vnodeInt.h | 17 +- source/dnode/vnode/src/meta/metaQuery.c | 78 +++++--- source/dnode/vnode/src/vnd/vnodeInitApi.c | 28 +-- source/libs/executor/src/sysscanoperator.c | 43 ++-- 5 files changed, 221 insertions(+), 163 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 4cfbf2c2fa..e2944d13da 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -16,13 +16,13 @@ #ifndef TDENGINE_STORAGEAPI_H #define TDENGINE_STORAGEAPI_H -#include "tsimplehash.h" -#include "tscalablebf.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tcommon.h" -#include "index.h" #include "function.h" +#include "index.h" +#include "taosdef.h" +#include "tcommon.h" +#include "tmsg.h" +#include "tscalablebf.h" +#include "tsimplehash.h" #ifdef __cplusplus extern "C" { @@ -46,7 +46,7 @@ typedef struct SMetaEntry { int8_t type; int8_t flags; // TODO: need refactor? tb_uid_t uid; - char * name; + char* name; union { struct { SSchemaWrapper schemaRow; @@ -57,43 +57,45 @@ typedef struct SMetaEntry { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char* comment; tb_uid_t suid; - uint8_t *pTags; + uint8_t* pTags; } ctbEntry; struct { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char* comment; int32_t ncid; // next column id SSchemaWrapper schemaRow; } ntbEntry; struct { - STSma *tsma; + STSma* tsma; } smaEntry; }; - uint8_t *pBuf; + uint8_t* pBuf; } SMetaEntry; typedef struct SMetaReader { - int32_t flags; - void * pMeta; - SDecoder coder; - SMetaEntry me; - void * pBuf; - int32_t szBuf; - struct SStoreMeta* pAPI; + int32_t flags; + void* pMeta; + SDecoder coder; + SMetaEntry me; + void* pBuf; + int32_t szBuf; + struct SStoreMeta* pAPI; } SMetaReader; typedef struct SMTbCursor { - void * pDbc; - void * pKey; - void * pVal; + void* pMeta; + void* pDbc; + void* pKey; + void* pVal; int32_t kLen; int32_t vLen; SMetaReader mr; + int8_t paused; } SMTbCursor; typedef struct SRowBuffPos { @@ -107,22 +109,22 @@ typedef struct SRowBuffPos { typedef struct SMetaTableInfo { int64_t suid; int64_t uid; - SSchemaWrapper *schema; + SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; typedef struct SSnapContext { - SMeta * pMeta; // todo remove it - int64_t snapVersion; - void * pCur; - int64_t suid; - int8_t subType; - SHashObj * idVersion; - SHashObj * suidInfo; - SArray * idList; - int32_t index; - bool withMeta; - bool queryMeta; // true-get meta, false-get data + SMeta* pMeta; // todo remove it + int64_t snapVersion; + void* pCur; + int64_t suid; + int8_t subType; + SHashObj* idVersion; + SHashObj* suidInfo; + SArray* idList; + int32_t index; + bool withMeta; + bool queryMeta; // true-get meta, false-get data } SSnapContext; typedef struct { @@ -139,10 +141,9 @@ typedef struct { // int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); -// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -// SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); -// int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); -// int32_t destroySnapContext(SSnapContext *ctx); +// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t +// *uid); SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext +// *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ @@ -219,16 +220,16 @@ typedef struct SStoreTqReader { bool (*tqReaderIsQueriedTable)(); bool (*tqReaderCurrentBlockConsumed)(); - struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it - int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it + struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it + int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { - int32_t (*createSnapshot)(SSnapContext *ctx, int64_t uid); - int32_t (*destroySnapshot)(SSnapContext *ctx); + int32_t (*createSnapshot)(SSnapContext* ctx, int64_t uid); + int32_t (*destroySnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); int32_t (*getTableInfoFromSnapshot)(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid); } SStoreSnapshotFn; @@ -252,42 +253,54 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t +payloadLen); */ typedef struct SStoreMeta { - SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor - void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor - int32_t (*cursorNext)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorNext - int32_t (*cursorPrev)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorPrev + SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor + void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor + void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first); // metaResumeTbCursor + int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev - int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); - int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList); - const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it + int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList); + int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList); + const void* (*extractTagVal)(const void* tag, int16_t type, STagVal* tagVal); // todo remove it - int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); - int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); - int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); - bool (*isTableExisted)(void *pVnode, tb_uid_t uid); + int32_t (*getTableUidByName)(void* pVnode, char* tbName, uint64_t* uid); + int32_t (*getTableTypeByName)(void* pVnode, char* tbName, ETableType* tbType); + int32_t (*getTableNameByUid)(void* pVnode, uint64_t uid, char* tbName); + bool (*isTableExisted)(void* pVnode, tb_uid_t uid); - int32_t (*metaGetCachedTbGroup)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); - int32_t (*metaPutTbGroupToCache)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); + int32_t (*metaGetCachedTbGroup)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); + int32_t (*metaPutTbGroupToCache)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); - int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes); - int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio); + int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, + bool* acquireRes); + int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen, double selectivityRatio); - void *(*storeGetIndexInfo)(); - void *(*getInvertIndex)(void* pVnode); - int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] - int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList - void *storeGetVersionRange; - void *storeGetLastTimestamp; + void* (*storeGetIndexInfo)(); + void* (*getInvertIndex)(void* pVnode); + int32_t (*getChildTableList)( + void* pVnode, int64_t suid, + SArray* list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList + void* storeGetVersionRange; + void* storeGetLastTimestamp; - int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema + int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema // db name, vgId, numOfTables, numOfSTables - int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); - void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + int32_t (*getNumOfChildTables)( + void* pVnode, int64_t uid, + int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, + int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & + // metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); int64_t (*getNumOfRowsInMem)(void* pVnode); /** @@ -298,24 +311,24 @@ int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); } SStoreMeta; typedef struct SStoreMetaReader { - void (*initReader)(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); - void (*clearReader)(SMetaReader *pReader); - void (*readerReleaseLock)(SMetaReader *pReader); - int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); - int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); - int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); + void (*initReader)(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI); + void (*clearReader)(SMetaReader* pReader); + void (*readerReleaseLock)(SMetaReader* pReader); + int32_t (*getTableEntryByUid)(SMetaReader* pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader* pReader, const char* name); + int32_t (*getEntryGetUidCache)(SMetaReader* pReader, tb_uid_t uid); } SStoreMetaReader; typedef struct SUpdateInfo { - SArray *pTsBuckets; + SArray* pTsBuckets; uint64_t numBuckets; - SArray *pTsSBFs; + SArray* pTsSBFs; uint64_t numSBFs; int64_t interval; int64_t watermark; TSKEY minTS; - SScalableBf *pCloseWinSBF; - SHashObj *pMap; + SScalableBf* pCloseWinSBF; + SHashObj* pMap; uint64_t maxDataVersion; } SUpdateInfo; @@ -334,15 +347,15 @@ typedef struct SStateStore { int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal); - void (*streamStateFreeVal)(void* val); + void (*streamStateFreeVal)(void* val); int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); - bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); + bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); int32_t (*streamStateClear)(SStreamState* pState); - void (*streamStateSetNumber)(SStreamState* pState, int32_t number); + void (*streamStateSetNumber)(SStreamState* pState, int32_t number); int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); @@ -353,36 +366,37 @@ typedef struct SStateStore { int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); - SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); - SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); - SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); - SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); - void (*streamStateFreeCur)(SStreamStateCur* pCur); + SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(SStreamStateCur* pCur); int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, + int32_t* pVLen); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); int32_t (*streamStateSessionClear)(SStreamState* pState); int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); - SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); - TSKEY (*updateInfoFillBlockData)(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); - bool (*updateInfoIsUpdated)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); - bool (*updateInfoIsTableInserted)(SUpdateInfo *pInfo, int64_t tbUid); - void (*updateInfoDestroy)(SUpdateInfo *pInfo); + SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); + void (*updateInfoDestroy)(SUpdateInfo* pInfo); - SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark); - void (*updateInfoAddCloseWindowSBF)(SUpdateInfo *pInfo); - void (*updateInfoDestoryColseWinSBF)(SUpdateInfo *pInfo); - int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); - int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, SUpdateInfo *pInfo); + SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark); + void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); + void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); @@ -396,11 +410,11 @@ typedef struct SStateStore { bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); - void (*streamStateClose)(SStreamState* pState, bool remove); - int32_t (*streamStateBegin)(SStreamState* pState); - int32_t (*streamStateCommit)(SStreamState* pState); - void (*streamStateDestroy)(SStreamState* pState, bool remove); - int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); + void (*streamStateClose)(SStreamState* pState, bool remove); + int32_t (*streamStateBegin)(SStreamState* pState); + int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateDestroy)(SStreamState* pState, bool remove); + int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); } SStateStore; typedef struct SStorageAPI { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index db285dc124..96486ec317 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -107,10 +107,12 @@ struct SQueryNode { typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; -SMTbCursor *metaOpenTbCursor(void *pVnode); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); +SMTbCursor* metaOpenTbCursor(void* pVnode); +void metaCloseTbCursor(SMTbCursor* pTbCur); +void metaPauseTbCursor(SMTbCursor* pTbCur); +void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first); +int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); #endif @@ -154,8 +156,8 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); -int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); @@ -175,7 +177,7 @@ void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -491,7 +493,6 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index fa9eea5e29..29fe89c3f2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,8 +17,8 @@ #include "osMemory.h" #include "tencode.h" -void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta *pAPI) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; metaReaderInit(pReader, pMeta, flags); pReader->pAPI = pAPI; } @@ -71,7 +71,7 @@ _err: } bool metaIsTableExist(void *pVnode, tb_uid_t uid) { - SVnode* pVnodeObj = pVnode; + SVnode *pVnodeObj = pVnode; metaRLock(pVnodeObj->pMeta); // query uid.idx if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { @@ -143,7 +143,7 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); @@ -179,7 +179,7 @@ int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { SMetaReader *pReader = &mr; // query name.idx - if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(((SMeta *)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; metaReaderClear(&mr); return -1; @@ -195,7 +195,7 @@ int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); code = metaGetTableEntryByName(&mr, tbName); if (code == 0) *tbType = mr.me.type; @@ -221,12 +221,13 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) { return NULL; } - SVnode* pVnodeObj = pVnode; - metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); + SVnode *pVnodeObj = pVnode; + // metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); - tdbTbcOpen(pVnodeObj->pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); - - tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + pTbCur->pMeta = pVnodeObj->pMeta; + pTbCur->paused = 1; + metaResumeTbCursor(pTbCur, 1); return pTbCur; } @@ -234,14 +235,45 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur) { tdbFree(pTbCur->pKey); tdbFree(pTbCur->pVal); - metaReaderClear(&pTbCur->mr); - if (pTbCur->pDbc) { - tdbTbcClose((TBC *)pTbCur->pDbc); + if (!pTbCur->paused) { + metaReaderClear(&pTbCur->mr); + if (pTbCur->pDbc) { + tdbTbcClose((TBC *)pTbCur->pDbc); + } } taosMemoryFree(pTbCur); } } +void metaPauseTbCursor(SMTbCursor *pTbCur) { + if (!pTbCur->paused) { + metaReaderClear(&pTbCur->mr); + tdbTbcClose((TBC *)pTbCur->pDbc); + pTbCur->paused = 1; + } +} +void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { + if (pTbCur->paused) { + metaReaderInit(&pTbCur->mr, pTbCur->pMeta, 0); + + tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + + if (first) { + tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + } else { + int c = 0; + tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); + if (c < 0) { + tdbTbcMoveToPrev(pTbCur->pDbc); + } else { + tdbTbcMoveToNext(pTbCur->pDbc); + } + } + + pTbCur->paused = 0; + } +} + int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { int ret; void *pBuf; @@ -974,7 +1006,7 @@ typedef struct { } SIdxCursor; int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; @@ -1034,7 +1066,7 @@ END: } int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1101,7 +1133,7 @@ END: return ret; } int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1132,7 +1164,7 @@ END: return 0; } int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; SMetaEntry oStbEntry = {0}; @@ -1318,7 +1350,7 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi } int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { - SMeta* pMeta = ((SVnode*) pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; const int32_t LIMIT = 128; int32_t isLock = false; @@ -1350,8 +1382,8 @@ int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { return 0; } -int32_t metaGetTableTags(void* pVnode, uint64_t suid, SArray *pUidTagInfo) { - SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode*)pVnode)->pMeta, suid, 1); +int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { + SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode *)pVnode)->pMeta, suid, 1); // If len > 0 means there already have uids, and we only want the // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept @@ -1456,11 +1488,11 @@ _exit: return code; } -int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t* numOfTables) { +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) { int32_t code = 0; *numOfTables = 0; - SVnode* pVnodeObj = pVnode; + SVnode *pVnodeObj = pVnode; metaRLock(pVnodeObj->pMeta); // fast path: search cache diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 526b9b4e2d..d2db6368a2 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "storageapi.h" -#include "vnodeInt.h" -#include "tstreamUpdate.h" #include "meta.h" +#include "storageapi.h" +#include "tstreamUpdate.h" +#include "vnodeInt.h" static void initTsdbReaderAPI(TsdReader* pReader); static void initMetadataAPI(SStoreMeta* pMeta); @@ -56,10 +56,10 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdReaderResetStatus = tsdbReaderReset; pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; - pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away pReader->tsdSetQueryTableList = tsdbSetTableList; - pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; + pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId; } void initMetadataAPI(SStoreMeta* pMeta) { @@ -67,6 +67,8 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->openTableMetaCursor = metaOpenTbCursor; pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->pauseTableMetaCursor = metaPauseTbCursor; + pMeta->resumeTableMetaCursor = metaResumeTbCursor; pMeta->cursorNext = metaTbCursorNext; pMeta->cursorPrev = metaTbCursorPrev; @@ -78,7 +80,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->storeGetIndexInfo = vnodeGetIdx; pMeta->getInvertIndex = vnodeGetIvtIdx; - pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + pMeta->extractTagVal = (const void* (*)(const void*, int16_t, STagVal*))metaGetTableTagVal; pMeta->getTableTags = metaGetTableTags; pMeta->getTableTagsByUid = metaGetTableTagsByUids; @@ -86,7 +88,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableTypeByName = metaGetTableTypeByName; pMeta->getTableNameByUid = metaGetTableNameByUid; - pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor + pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor pMeta->storeGetTableList = vnodeGetTableList; pMeta->getCachedTableList = metaGetCachedTableUidList; @@ -106,7 +108,7 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderNextBlockInWal = tqNextBlockInWal; - pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it + pTq->tqNextBlockImpl = tqNextBlockImpl; // todo remove it pTq->tqReaderAddTables = tqReaderAddTbUidList; pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; @@ -116,10 +118,10 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; - pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it - pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it - pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqGetResultBlock = tqGetResultBlock; pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; @@ -199,7 +201,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateClose = streamStateClose; pStore->streamStateBegin = streamStateBegin; pStore->streamStateCommit = streamStateCommit; - pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDestroy = streamStateDestroy; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; } @@ -239,4 +241,4 @@ void initSnapshotFn(SStoreSnapshotFn* pSnapshot) { pSnapshot->destroySnapshot = destroySnapContext; pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot; pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index fd766e21f8..23a7d2c9e9 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -25,6 +25,7 @@ #include "tdatablock.h" #include "tmsg.h" +#include "index.h" #include "operator.h" #include "query.h" #include "querytask.h" @@ -32,7 +33,6 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "index.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); @@ -540,12 +540,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t restore = pInfo->restore; pInfo->restore = false; - + while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { if (restore) { restore = false; } - + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -626,8 +626,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { @@ -1100,8 +1100,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { } static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; @@ -1288,11 +1288,16 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int8_t firstMetaCursor = 0; SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + firstMetaCursor = 1; + } + if (!firstMetaCursor) { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } blockDataCleanup(pInfo->pRes); @@ -1436,12 +1441,14 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { numOfRows = 0; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } } } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); p->info.rows = numOfRows; pInfo->pRes->info.rows = numOfRows; @@ -1485,7 +1492,8 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; + SSTabFltArg arg = { + .pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; @@ -1827,7 +1835,7 @@ void destroySysScanOperator(void* param) { pInfo->pIdx = NULL; } - if(pInfo->pSchema) { + if (pInfo->pSchema) { taosHashCleanup(pInfo->pSchema); pInfo->pSchema = NULL; } @@ -2144,7 +2152,7 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { return -1; } -static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { +static int32_t doGetTableRowSize(SReadHandle* pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { *rowLen = 0; SMetaReader mr = {0}; @@ -2194,17 +2202,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SBlockDistInfo* pBlockScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, - (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, + GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t) pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); + blockDistInfo.numOfInmemRows = (int32_t)pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2289,7 +2297,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, + (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; @@ -2316,4 +2325,4 @@ _error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; -} \ No newline at end of file +} From 046c9e9494b95f945920fbd61dce91685929548d Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Mon, 5 Jun 2023 15:32:28 +0800 Subject: [PATCH 038/122] docs: go connector add tmq assignment and seek --- docs/en/14-reference/03-connector/05-go.mdx | 22 +++++++++++++++++++-- docs/zh/08-connector/20-go.mdx | 22 +++++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index 0088f23006..7e5023b6a7 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -379,6 +379,15 @@ Note: `tmq.TopicPartition` is reserved for compatibility purpose Commit information. +* `func (c *Consumer) Assignment() (partitions []tmq.TopicPartition, err error)` + +Get Assignment(TDengine >= 3.0.5.0 and driver-go >= v3.5.0 are required). + +* `func (c *Consumer) Seek(partition tmq.TopicPartition, ignoredTimeoutMs int) error` +Note: `ignoredTimeoutMs` is reserved for compatibility purpose + +Seek offset(TDengine >= 3.0.5.0 and driver-go >= v3.5.0 are required). + * `func (c *Consumer) Unsubscribe() error` Unsubscribe. @@ -468,6 +477,15 @@ Note: `tmq.TopicPartition` is reserved for compatibility purpose Commit information. +* `func (c *Consumer) Assignment() (partitions []tmq.TopicPartition, err error)` + +Get Assignment(TDengine >= 3.0.5.0 and driver-go >= v3.5.0 are required). + +* `func (c *Consumer) Seek(partition tmq.TopicPartition, ignoredTimeoutMs int) error` +Note: `ignoredTimeoutMs` is reserved for compatibility purpose + +Seek offset(TDengine >= 3.0.5.0 and driver-go >= v3.5.0 are required). + * `func (c *Consumer) Unsubscribe() error` Unsubscribe. @@ -476,7 +494,7 @@ Unsubscribe. Close consumer. -For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) +For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/main/examples/tmqoverws/main.go) ### parameter binding via WebSocket @@ -524,7 +542,7 @@ For a complete example see [GitHub sample file](https://github.com/taosdata/driv Closes the parameter binding. -For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go) +For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/main/examples/stmtoverws/main.go) ## API Reference diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index fd6df992b5..5461328182 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -383,6 +383,15 @@ func main() { 提交消息。 +* `func (c *Consumer) Assignment() (partitions []tmq.TopicPartition, err error)` + + 获取消费进度。(需要 TDengine >= 3.0.5.0， driver-go >= v3.5.0) + +* `func (c *Consumer) Seek(partition tmq.TopicPartition, ignoredTimeoutMs int) error` +注意：出于兼容目的保留 `ignoredTimeoutMs` 参数，当前未使用 + + 按照指定的进度消费。(需要 TDengine >= 3.0.5.0， driver-go >= v3.5.0) + * `func (c *Consumer) Close() error` 关闭连接。 @@ -468,11 +477,20 @@ func main() { 提交消息。 +* `func (c *Consumer) Assignment() (partitions []tmq.TopicPartition, err error)` + + 获取消费进度。(需要 TDengine >= 3.0.5.0， driver-go >= v3.5.0) + +* `func (c *Consumer) Seek(partition tmq.TopicPartition, ignoredTimeoutMs int) error` +注意：出于兼容目的保留 `ignoredTimeoutMs` 参数，当前未使用 + + 按照指定的进度消费。(需要 TDengine >= 3.0.5.0， driver-go >= v3.5.0) + * `func (c *Consumer) Close() error` 关闭连接。 -完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) +完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/main/examples/tmqoverws/main.go) ### 通过 WebSocket 进行参数绑定 @@ -520,7 +538,7 @@ func main() { 结束参数绑定。 -完整参数绑定示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go) +完整参数绑定示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/main/examples/stmtoverws/main.go) ## API 参考 From 6b670b7f0801b490e7e5e66e25428be233529eb1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 5 Jun 2023 15:48:15 +0800 Subject: [PATCH 039/122] fix desc --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 57f708c5cd..be3c41e2ee 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3034,7 +3034,7 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); } if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { - code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant"); + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant"); } else if (TSDB_CODE_SUCCESS != code) { code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); } From 005182c3c925169d45392feb0755beaf50efca2a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 5 Jun 2023 16:40:48 +0800 Subject: [PATCH 040/122] fix multiple interp issue --- source/libs/parser/src/parTranslater.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index be3c41e2ee..f049af6747 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3041,7 +3041,7 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList return code; } -static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) { +static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList, bool isInterpFill) { if (FILL_MODE_VALUE != pFill->mode && FILL_MODE_VALUE_F != pFill->mode) { return TSDB_CODE_SUCCESS; } @@ -3058,10 +3058,13 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (TSDB_CODE_SUCCESS != code) { return code; } - ++fillNo; + + if (!isInterpFill) { + ++fillNo; + } } } - if (fillNo != LIST_LENGTH(pFillValues->pNodeList)) { + if (!isInterpFill && fillNo != LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } return TSDB_CODE_SUCCESS; @@ -3072,7 +3075,7 @@ static int32_t translateFillValues(STranslateContext* pCxt, SSelectStmt* pSelect NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { return TSDB_CODE_SUCCESS; } - return checkFillValues(pCxt, (SFillNode*)((SIntervalWindowNode*)pSelect->pWindow)->pFill, pSelect->pProjectionList); + return checkFillValues(pCxt, (SFillNode*)((SIntervalWindowNode*)pSelect->pWindow)->pFill, pSelect->pProjectionList, false); } static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { @@ -3522,7 +3525,7 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true); } if (TSDB_CODE_SUCCESS == code) { - code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList); + code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList, true); } return code; From 384aa4d70bc1751037ec9af1be813eba490af5ad Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 5 Jun 2023 16:41:09 +0800 Subject: [PATCH 041/122] add test cases --- tests/system-test/2-query/interp.py | 128 ++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 121d4dcff6..019b449dfd 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -219,6 +219,56 @@ class TDTestCase: tdSql.checkData(2, 0, 12) tdSql.checkData(3, 0, 12) + ## test fill value with scalar expression + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1.0 + 2.0)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2.5)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + '2')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + '2.0')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '3' + 'abc')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '2' + '1abc')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + tdLog.printNoPrefix("==========step5:fill prev") ## {. . .} @@ -1837,6 +1887,84 @@ class TDTestCase: tdSql.checkData(59, 1, 123) tdSql.checkData(60, 1, 60) # + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123 + 123)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, 246) + tdSql.checkData(4, 0, 246) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, 246) + tdSql.checkData(9, 0, 246) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, 246) + tdSql.checkData(14, 0, 246) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, 246) + tdSql.checkData(19, 0, 246) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, 246) + tdSql.checkData(24, 0, 246) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, 246) + tdSql.checkData(29, 0, 246) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 246) + tdSql.checkData(34, 0, 246) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 246) + tdSql.checkData(39, 0, 246) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, 246) + tdSql.checkData(44, 0, 246) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, 246) + tdSql.checkData(49, 0, 246) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, 246) + tdSql.checkData(54, 0, 246) + tdSql.checkData(55, 0, None) # + tdSql.checkData(59, 0, 246) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, 246) + tdSql.checkData(4, 1, 246) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, 246) + tdSql.checkData(9, 1, 246) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, 246) + tdSql.checkData(14, 1, 246) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, 246) + tdSql.checkData(19, 1, 246) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, 246) + tdSql.checkData(24, 1, 246) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, 246) + tdSql.checkData(29, 1, 246) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, 246) + tdSql.checkData(34, 1, 246) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, 246) + tdSql.checkData(39, 1, 246) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 246) + tdSql.checkData(44, 1, 246) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, 246) + tdSql.checkData(49, 1, 246) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, 246) + tdSql.checkData(54, 1, 246) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, 246) + tdSql.checkData(59, 1, 246) + tdSql.checkData(60, 1, 60) # + # test fill prev tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(prev)") tdSql.checkRows(61) From cb024ca069bc0f8852171ac950f9d41b45fd73c0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 5 Jun 2023 10:52:07 +0000 Subject: [PATCH 042/122] fix invalid free --- source/libs/stream/src/tstreamFileState.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index bc84509728..bfaeca89f6 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -419,7 +419,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; } - memcpy(val, buf, len); + memcpy(buf, val, len); buf[len] = 0; maxCheckPointId = atol((char*)buf); taosMemoryFree(val); @@ -433,7 +433,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0) { return TSDB_CODE_FAILED; } - memcpy(val, buf, len); + memcpy(buf, val, len); buf[len] = 0; taosMemoryFree(val); From 02b1257b01a3277d20f658f5fb9bb865c5d85af2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 6 Jun 2023 11:17:02 +0800 Subject: [PATCH 043/122] enh: reassign data column in blocks --- include/common/tcommon.h | 1 + source/common/src/tdatablock.c | 76 +++++++++++++++++++++++--- source/libs/executor/src/executorInt.c | 1 - source/libs/function/src/tudf.c | 16 +++++- 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 2f93f8c3e3..d2352e100c 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -231,6 +231,7 @@ typedef struct SColumnInfoData { }; SColumnInfo info; // column info bool hasNull; // if current column data has null value. + bool reassigned; // if current column data is reassigned. } SColumnInfoData; typedef struct SQueryTableDataCond { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a6a54a8347..24e978b0ea 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -23,6 +23,20 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + if (pColumnInfoData->reassigned) { + int32_t totalSize = 0; + for (int32_t row = 0; row < numOfRows; ++row) { + char* pColData = pColumnInfoData->pData + pColumnInfoData->varmeta.offset[row]; + int32_t colSize = 0; + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + totalSize += colSize; + } + return totalSize; + } return pColumnInfoData->varmeta.length; } else { if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) { @@ -138,8 +152,8 @@ int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, SVarColAttr* pAttr = &pColumnInfoData->varmeta; - uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[dstRowIdx] = pColumnInfoData->varmeta.offset[srcRowIdx]; + pColumnInfoData->reassigned = true; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * dstRowIdx, pData, pColumnInfoData->info.bytes); colDataClearNull_f(pColumnInfoData->nullbitmap, dstRowIdx); @@ -603,8 +617,22 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { *(int32_t*)pStart = dataSize; pStart += sizeof(int32_t); - memcpy(pStart, pCol->pData, dataSize); - pStart += dataSize; + if (pCol->reassigned && IS_VAR_DATA_TYPE(pCol->info.type)) { + for (int32_t row = 0; row < numOfRows; ++row) { + char* pColData = pCol->pData + pCol->varmeta.offset[row]; + int32_t colSize = 0; + if (pCol->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + memcpy(pStart, pColData, colSize); + pStart += colSize; + } + } else { + memcpy(pStart, pCol->pData, dataSize); + pStart += dataSize; + } } return 0; @@ -1764,7 +1792,20 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int32_t len = colDataGetLength(pColData, rows); tlen += taosEncodeFixedI32(buf, len); - tlen += taosEncodeBinary(buf, pColData->pData, len); + if (pColData->reassigned && IS_VAR_DATA_TYPE(pColData->info.type)) { + for (int32_t row = 0; row < rows; ++row) { + char* pData = pColData->pData + pColData->varmeta.offset[row]; + int32_t colSize = 0; + if (pColData->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pData); + } else { + colSize = varDataTLen(pData); + } + tlen += taosEncodeBinary(buf, pData, colSize); + } + } else { + tlen += taosEncodeBinary(buf, pColData->pData, len); + } } return tlen; } @@ -2525,12 +2566,29 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += metaSize; dataLen += metaSize; - colSizes[col] = colDataGetLength(pColRes, numOfRows); - dataLen += colSizes[col]; - if (pColRes->pData != NULL) { - memmove(data, pColRes->pData, colSizes[col]); + if (pColRes->reassigned && IS_VAR_DATA_TYPE(pColRes->info.type)) { + colSizes[col] = 0; + for (int32_t row = 0; row < numOfRows; ++row) { + char* pColData = pColRes->pData + pColRes->varmeta.offset[row]; + int32_t colSize = 0; + if (pColRes->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + colSizes[col] += colSize; + dataLen += colSize; + memmove(data, pColData, colSize); + data += colSize; + } + } else { + colSizes[col] = colDataGetLength(pColRes, numOfRows); + dataLen += colSizes[col]; + if (pColRes->pData != NULL) { + memmove(data, pColRes->pData, colSizes[col]); + } + data += colSizes[col]; } - data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, htonl(colSizes[col]), colSizes[col]); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index ce7a117543..fbc0512a26 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -562,7 +562,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD int32_t numOfRows = 0; if (IS_VAR_DATA_TYPE(pDst->info.type)) { int32_t j = 0; - pDst->varmeta.length = 0; while (j < totalRows) { if (pIndicator[j] == 0) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 6b70422ac8..31a7dfdbc5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -791,7 +791,21 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen); udfCol->colData.varLenCol.payloadLen = colDataGetLength(col, udfBlock->numOfRows); udfCol->colData.varLenCol.payload = taosMemoryMalloc(udfCol->colData.varLenCol.payloadLen); - memcpy(udfCol->colData.varLenCol.payload, col->pData, udfCol->colData.varLenCol.payloadLen); + if (col->reassigned) { + for (int32_t row = 0; row < udfCol->colData.numOfRows; ++row) { + char* pColData = col->pData + col->varmeta.offset[row]; + int32_t colSize = 0; + if (col->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + memcpy(udfCol->colData.varLenCol.payload, pColData, colSize); + udfCol->colData.varLenCol.payload += colSize; + } + } else { + memcpy(udfCol->colData.varLenCol.payload, col->pData, udfCol->colData.varLenCol.payloadLen); + } } else { udfCol->colData.fixLenCol.nullBitmapLen = BitmapLen(udfCol->colData.numOfRows); int32_t bitmapLen = udfCol->colData.fixLenCol.nullBitmapLen; From 472665e4b16f77d1355f77333dc79c4d39c11600 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 6 Jun 2023 13:06:53 +0800 Subject: [PATCH 044/122] fix: compress core by tDataTypes --- include/util/tdef.h | 2 +- source/common/src/ttypes.c | 10 +++++++++- source/libs/parser/src/parUtil.c | 2 +- source/util/src/terror.c | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 0b0569e2d1..37eeb87fdd 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -32,7 +32,7 @@ extern "C" { #define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle // Bytes for each type. -extern const int32_t TYPE_BYTES[17]; +extern const int32_t TYPE_BYTES[21]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 33b972594e..39255cff3a 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -17,7 +17,7 @@ #include "ttypes.h" #include "tcompression.h" -const int32_t TYPE_BYTES[17] = { +const int32_t TYPE_BYTES[21] = { -1, // TSDB_DATA_TYPE_NULL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT @@ -34,6 +34,10 @@ const int32_t TYPE_BYTES[17] = { INT_BYTES, // TSDB_DATA_TYPE_UINT sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_VARBINARY: placeholder, not implemented + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_BLOB: placeholder, not implemented + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_MEDIUMBLOB: placeholder, not implemented sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_GEOMETRY }; @@ -57,6 +61,10 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, + {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString}, }; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 4a51d3c31c..6aed29b581 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -138,7 +138,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY: return "Primary timestamp column cannot be dropped"; case TSDB_CODE_PAR_INVALID_MODIFY_COL: - return "Only binary/nchar column length could be modified, and the length can only be increased, not decreased"; + return "Only binary/nchar/geometry column length could be modified, and the length can only be increased, not decreased"; case TSDB_CODE_PAR_INVALID_TBNAME: return "Invalid tbname pseudo column"; case TSDB_CODE_PAR_INVALID_FUNCTION_NAME: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e28e67f83a..7a900269e8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -542,7 +542,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TIMELINE_FUNC, "Invalid timeline fu TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PASSWD, "Invalid password") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Invalid alter table statement") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY, "Primary timestamp column cannot be dropped") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar column length could be modified, and the length can only be increased, not decreased") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar/geometry column length could be modified, and the length can only be increased, not decreased") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TBNAME, "Invalid tbname pseudo column") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FUNCTION_NAME, "Invalid function name") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COMMENT_TOO_LONG, "Comment too long") From b65df61efd04da69aed29444d306844c4bc2b740 Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 6 Jun 2023 15:42:28 +0800 Subject: [PATCH 045/122] docs: add consuemr demo doc --- docs/en/14-reference/03-connector/04-java.mdx | 1 + docs/zh/08-connector/14-java.mdx | 1 + examples/JDBC/consumer-demo/readme.md | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 114026eca0..db49e5f395 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -1256,6 +1256,7 @@ The source code of the sample application is under `TDengine/examples/JDBC`: - connectionPools: using taos-jdbcdriver in connection pools such as HikariCP, Druid, dbcp, c3p0, etc. - SpringJdbcTemplate: using taos-jdbcdriver in Spring JdbcTemplate. - mybatisplus-demo: using taos-jdbcdriver in Springboot + Mybatis. +- consumer-demo: consumer TDengine data example, the consumption rate can be controlled by parameters. [JDBC example](https://github.com/taosdata/TDengine/tree/3.0/examples/JDBC) diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index e4cf4a83e7..46800226d7 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -1258,6 +1258,7 @@ public static void main(String[] args) throws Exception { - connectionPools：HikariCP, Druid, dbcp, c3p0 等连接池中使用 taos-jdbcdriver。 - SpringJdbcTemplate：Spring JdbcTemplate 中使用 taos-jdbcdriver。 - mybatisplus-demo：Springboot + Mybatis 中使用 taos-jdbcdriver。 +- consumer-demo：Consumer 消费 TDengine 数据示例，可通过参数控制消费速度。 请参考：[JDBC example](https://github.com/taosdata/TDengine/tree/3.0/examples/JDBC) diff --git a/examples/JDBC/consumer-demo/readme.md b/examples/JDBC/consumer-demo/readme.md index 77742ab605..c211b017a7 100644 --- a/examples/JDBC/consumer-demo/readme.md +++ b/examples/JDBC/consumer-demo/readme.md @@ -4,7 +4,7 @@ TDengine's Consumer demo project is organized in a Maven way so that users can e sudo apt-get install maven ``` -## Install TDengine Client +## Install TDengine Client and TaosAdapter Make sure you have already installed a tdengine client on your current develop environment. Download the tdengine package on our website: ``https://www.taosdata.com/cn/all-downloads/`` and install the client. From 201d0f2d18c6a3b3ff77df64e8677f5f6e7f264c Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Tue, 6 Jun 2023 15:35:39 +0800 Subject: [PATCH 046/122] fix stream ci issue --- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 50 ++++++++++--- .../tsim/stream/distributeInterval0.sim | 71 ++++++++++++------- 4 files changed, 90 insertions(+), 34 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ffc63a22a8..38890f8c34 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -457,6 +457,7 @@ typedef struct SStreamIntervalOperatorInfo { int64_t dataVersion; SStateStore statestore; bool recvGetAll; + SHashObj* pFinalPullDataMap; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c93b9f4c73..2702cf6861 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1865,7 +1865,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { - pInfo->pUpdateInfo->maxDataVersion = pTaskInfo->streamInfo.fillHistoryVer2; + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pTaskInfo->streamInfo.fillHistoryVer2); doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); } } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 3a83472079..d6429fd121 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1306,6 +1306,8 @@ static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) return true; } +static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } + static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; @@ -1340,8 +1342,14 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); if (chIds) { - getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); - continue; + int32_t childId = getChildIndex(pBlock); + SArray* chArray = *(void**)chIds; + int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); + if (index != -1) { + qDebug("===stream===try push delete window%" PRId64 "chId:%d ,continue", win.skey, childId); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); + continue; + } } bool res = doDeleteWindow(pOperator, win.skey, winGpId); if (pUpWins && res) { @@ -1497,6 +1505,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { taosArrayDestroy(*(void**)pIte); } taosHashCleanup(pInfo->pPullDataMap); + taosHashCleanup(pInfo->pFinalPullDataMap); taosArrayDestroy(pInfo->pPullWins); blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); @@ -2067,8 +2076,6 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*)); } -static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } - static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); @@ -2112,7 +2119,7 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) { +void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); @@ -2136,6 +2143,22 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); qDebug("===stream===retrive pull data over.window %" PRId64 , winRes.ts); + + void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); + if (pFinalCh) { + taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); + doDeleteWindow(pOperator, winRes.ts, winRes.groupId); + STimeWindow nextWin = getFinalTimeWindow(winRes.ts, pInterval); + SPullWindowInfo pull = {.window = nextWin, + .groupId = winRes.groupId, + .calWin.skey = nextWin.skey, + .calWin.ekey = nextWin.skey}; + // add pull data request + if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) { + addPullWindow(pMap, &winRes, numOfCh); + qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); + } + } } } } @@ -2144,7 +2167,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) } } -static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { +static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); @@ -2161,6 +2184,14 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } + } else { + SArray* chArray = *(void**)chIds; + int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); + qDebug("===stream===check final retrive %" PRId64",chid:%d", winKey->ts, index); + if (index == -1) { + qDebug("===stream===add final retrive %" PRId64, winKey->ts); + taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0); + } } } } @@ -2554,7 +2585,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { - addRetriveWindow(delWins, pInfo); + int32_t chId = getChildIndex(pBlock); + addRetriveWindow(delWins, pInfo, chId); if (pBlock->info.type != STREAM_CLEAR) { taosArrayAddAll(pInfo->pDelWins, delWins); } @@ -2589,7 +2621,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) { - processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval); + processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, pInfo->numOfChild, pOperator); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -2772,6 +2804,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; @@ -4963,6 +4996,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pPhyNode = NULL; // create new child pInfo->pPullDataMap = NULL; + pInfo->pFinalPullDataMap = NULL; pInfo->pPullWins = NULL; // SPullWindowInfo pInfo->pullIndex = 0; pInfo->pPullDataRes = NULL; diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 959b32fa59..5bb03c8cbf 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -1,36 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 system sh/exec.sh -n dnode1 -s start sleep 50 sql connect -sql create dnode $hostname2 port 7200 -system sh/exec.sh -n dnode2 -s start - -print ===== step1 -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - print ====> dnode not ready! - return -1 - endi -sql select * from information_schema.ins_dnodes -print ===> $data00 $data01 $data02 $data03 $data04 $data05 -print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then - return -1 -endi -if $data(1)[4] != ready then - goto step1 -endi -if $data(2)[4] != ready then - goto step1 -endi print ===== step2 sql drop stream if exists stream_t1; @@ -248,10 +223,56 @@ sql insert into ts3 values(1648791223002,2,2,3,1.1); sql insert into ts4 values(1648791233003,3,2,3,2.1); sql insert into ts3 values(1648791243004,4,2,43,73.1); sql insert into ts4 values(1648791213002,24,22,23,4.1); + +$loop_count = 0 +loop032: + +$loop_count = $loop_count + 1 +if $loop_count == 30 then + return -1 +endi + +sleep 1000 +print 6-0 select * from streamtST1; +sql select * from streamtST1; + +if $rows != 4 then + print =====rows=$rows + goto loop032 +endi + +if $data01 != 8 then + print =6====data01=$data01 + goto loop032 +endi + sql insert into ts3 values(1648791243005,4,20,3,3.1); sql insert into ts4 values(1648791243006,4,2,3,3.1) (1648791243007,4,2,3,3.1) ; sql insert into ts3 values(1648791243008,4,2,30,3.1) (1648791243009,4,2,3,3.1) (1648791243010,4,2,3,3.1) ; sql insert into ts4 values(1648791243011,4,2,3,3.1) (1648791243012,34,32,33,3.1) (1648791243013,4,2,3,3.1) (1648791243014,4,2,13,3.1); + +$loop_count = 0 +loop033: + +$loop_count = $loop_count + 1 +if $loop_count == 30 then + return -1 +endi + +sleep 1000 +print 6-1 select * from streamtST1; +sql select * from streamtST1; + +if $rows != 4 then + print =====rows=$rows + goto loop033 +endi + +if $data01 != 8 then + print =6====data01=$data01 + goto loop033 +endi + sql insert into ts3 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; $loop_count = 0 From bcac24b6b7ede099377b99cb85ca14e4bf53d848 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 6 Jun 2023 17:19:59 +0800 Subject: [PATCH 047/122] fix: tsma query with order by _wstart/_wend --- source/libs/parser/src/parTranslater.c | 1 + source/libs/planner/src/planOptimizer.c | 23 +++++- .../script/tsim/sma/tsmaCreateInsertQuery.sim | 74 +++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d1c3da0041..a9fe65492d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4940,6 +4940,7 @@ static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); + snprintf(pTable->table.tableAlias, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); *pOutput = (SNode*)pTable; return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8b75fe6b33..36378a5414 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1385,7 +1385,18 @@ static SNode* smaIndexOptFindWStartFunc(SNodeList* pSmaFuncs) { return NULL; } -static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, +static SNode* smaIndexOptFuncInProject(SNodeList* pProjects, SFunctionNode* pFunc) { + SNode* pProject = NULL; + FOREACH(pProject, pProjects) { + if (0 != pFunc->node.aliasName[0] && + 0 == strncmp(pFunc->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { + return pProject; + } + } + return NULL; +} + +static int32_t smaIndexOptCreateSmaCols(SWindowLogicNode* pWindow, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput) { SNodeList* pCols = NULL; SNode* pFunc = NULL; @@ -1393,11 +1404,16 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo int32_t index = 0; int32_t smaFuncIndex = -1; bool hasWStart = false; - FOREACH(pFunc, pFuncs) { + + SProjectLogicNode* pProject = (SProjectLogicNode*)pWindow->node.pParent; + FOREACH(pFunc, pWindow->pFuncs) { smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; } else { + if (pProject && !smaIndexOptFuncInProject(pProject->pProjections, (SFunctionNode*)pFunc)) { + continue; + } code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 1)); if (TSDB_CODE_SUCCESS != code) { break; @@ -1444,10 +1460,11 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; } + SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols); + code = smaIndexOptCreateSmaCols(pWindow, pIndex->dstTbUid, pSmaFuncs, pCols); } nodesDestroyList(pSmaFuncs); return code; diff --git a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim index 242231e408..60f769d2ae 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim @@ -340,6 +340,80 @@ if $data05 != 30.000000000 then return -1 endi +print =============== select with _wstart/order by _wstart from stb from file in designated vgroup +sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m) order by _wstart; +print $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data02 != -13 then + print data02 $data02 != -13 + return -1 +endi + +if $data03 != 20.00000 then + print data03 $data03 != 20.00000 + return -1 +endi + +if $data04 != 20 then + print data04 $data04 != 20 + return -1 +endi + +print =============== select without _wstart/with order by _wstart from stb from file in designated vgroup +sql select _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m) order by _wstart; +print $data00 $data01 $data02 $data03 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != -13 then + print data01 $data01 != -13 + return -1 +endi + +if $data02 != 20.00000 then + print data02 $data02 != 20.00000 + return -1 +endi + +if $data03 != 20 then + print data03 $data03 != 20 + return -1 +endi + +print =============== select * from stb from file in common vgroups +sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m) order by _wstart; +print $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data02 != -13 then + print data02 $data02 != -13 + return -1 +endi + +if $data03 != 20.00000 then + print data03 $data03 != 20.00000 + return -1 +endi + +if $data04 != 20 then + print data04 $data04 != 20 + return -1 +endi + +if $data05 != 30.000000000 then + print data05 $data05 != 30.000000000 + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 44ed6f0c92b5ee4e12ead7edc23d033b11609f9c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 6 Jun 2023 18:51:08 +0800 Subject: [PATCH 048/122] release: update default version --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 6d893c0627..3d0dc80902 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.4.3") + SET(TD_VER_NUMBER "3.0.5.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index c7836d1298..a9336697f2 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.5.0 + + + ## 3.0.4.2 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 2b28bae745..3ee19de84f 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.5.0 + + + ## 3.0.4.2 From 3cb64f3775e23c72fe29926dc4302c8c8cc37362 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 6 Jun 2023 19:28:55 +0800 Subject: [PATCH 049/122] fix: prevent projectoperator scanning all rows when limit with no group --- source/libs/executor/src/groupoperator.c | 2 ++ source/libs/executor/src/joinoperator.c | 5 +++++ source/libs/executor/src/projectoperator.c | 2 ++ 3 files changed, 9 insertions(+) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 7aac639027..c448ea0160 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -647,6 +647,8 @@ uint64_t calcGroupId(char* pData, int32_t len) { // NOTE: only extract the initial 8 bytes of the final MD5 digest uint64_t id = 0; memcpy(&id, context.digest, sizeof(uint64_t)); + if (0 == id) + memcpy(&id, context.digest + 8, sizeof(uint64_t)); return id; } diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f4737..744e513b96 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -183,6 +183,11 @@ void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; nodesDestroyNode(pJoinOperator->pCondAfterMerge); + taosArrayDestroy(pJoinOperator->rowCtx.leftCreatedBlocks); + taosArrayDestroy(pJoinOperator->rowCtx.rightCreatedBlocks); + taosArrayDestroy(pJoinOperator->rowCtx.leftRowLocations); + taosArrayDestroy(pJoinOperator->rowCtx.rightRowLocations); + pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index dde6f7c0e8..e7de826d4b 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -213,6 +213,8 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS } else { if (limitReached && (pLimitInfo->slimit.limit >= 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { setOperatorCompleted(pOperator); + } else if (limitReached && groupId == 0) { + setOperatorCompleted(pOperator); } } From b2029e79a3a4dc6a15f8b8001a33fe30bc0d0f75 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 6 Jun 2023 19:55:44 +0800 Subject: [PATCH 050/122] update taos-tools release version --- docs/en/28-releases/02-tools.md | 4 ++++ docs/zh/28-releases/02-tools.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 9f8dbfee7e..28c2ff7a7f 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.5.1 + + + ## 2.5.0 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 7f93483ed4..fbd12b1440 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.5.1 + + + ## 2.5.0 From 2ab43ca0d009b9ddcee3153da3fc3f771a3c2520 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Tue, 6 Jun 2023 17:37:09 +0800 Subject: [PATCH 051/122] semi interval op check data ts --- source/libs/executor/src/timewindowoperator.c | 2 +- tests/script/tsim/stream/ignoreExpiredData.sim | 4 ++-- tests/script/tsim/stream/sliding.sim | 7 ------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d6429fd121..2676e097f9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2345,7 +2345,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + if ((pInfo->ignoreExpiredData && isClosed && !IS_FINAL_OP(pInfo)) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); if (startPos < 0) { break; diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim index 27920dd539..884b7cbb5f 100644 --- a/tests/script/tsim/stream/ignoreExpiredData.sim +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -132,12 +132,12 @@ if $loop_count == 10 then return -1 endi -if $data01 != 1 then +if $data01 != 2 then print =====data01=$data01 goto loop4 endi -if $data02 != 1 then +if $data02 != 2 then print =====data02=$data02 goto loop4 endi diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 3312ccbec4..05eb7dacba 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -576,13 +576,6 @@ $loop_count = 0 print step 7 -loop4: -sleep 100 - -$loop_count = $loop_count + 1 -if $loop_count == 10 then - return -1 -endi sql create database test3 vgroups 6; sql use test3; From 1e9c4d5facfaf7dbae9c2d6c30636af6193e563c Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Wed, 7 Jun 2023 10:45:27 +0800 Subject: [PATCH 052/122] opt stream block dispatch --- include/common/tdatablock.h | 1 + include/libs/stream/tstream.h | 1 + source/common/src/tdatablock.c | 24 +++++++--- source/dnode/vnode/src/tq/tqSink.c | 6 +-- source/libs/stream/src/streamDispatch.c | 63 ++++++++++++++++++------- source/libs/stream/src/streamTask.c | 4 ++ 6 files changed, 71 insertions(+), 28 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 53fc07c3f3..6cb7d88523 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -248,6 +248,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); +int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8316e6ef50..51f2de481d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -327,6 +327,7 @@ struct SStreamTask { int64_t checkpointingId; int32_t checkpointAlignCnt; struct SStreamMeta* pMeta; + SSHashObj* pNameMap; }; // meta diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 24e978b0ea..033fbb0ef1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2465,19 +2465,31 @@ _end: } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { - if (stbFullName[0] == 0) { + char* pBuf = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); + if (!pBuf) { return NULL; } + int32_t code = buildCtbNameByGroupIdImpl(stbFullName, groupId, pBuf); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pBuf); + return NULL; + } + return pBuf; +} + +int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) { + if (stbFullName[0] == 0) { + return TSDB_CODE_FAILED; + } SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); if (tags == NULL) { - return NULL; + return TSDB_CODE_FAILED; } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - return NULL; + return TSDB_CODE_FAILED; } SSmlKv pTag = {.key = "group_id", @@ -2499,9 +2511,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { - return NULL; + return TSDB_CODE_FAILED; } - return rname.ctbShortName; + return TSDB_CODE_SUCCESS; } int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index db1b5ed902..9349c6eb0d 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -298,10 +298,8 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d if (res == TSDB_CODE_SUCCESS) { memcpy(ctbName, pTableSinkInfo->tbName, strlen(pTableSinkInfo->tbName)); } else { - char* tmp = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); - memcpy(ctbName, tmp, strlen(tmp)); - memcpy(pTableSinkInfo->tbName, tmp, strlen(tmp)); - taosMemoryFree(tmp); + buildCtbNameByGroupIdImpl(stbFullName, pDataBlock->info.id.groupId, ctbName); + memcpy(pTableSinkInfo->tbName, ctbName, strlen(ctbName)); tqDebug("vgId:%d, gropuId:%" PRIu64 " datablock table name is null", TD_VID(pVnode), pDataBlock->info.id.groupId); } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9cb0a56644..922a1f5345 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -15,6 +15,13 @@ #include "streamInc.h" +#define MAX_BLOCK_NAME_NUM 1024 + +typedef struct SBlockName { + uint32_t hashValue; + char parTbName[TSDB_TABLE_NAME_LEN]; +} SBlockName; + int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -331,26 +338,46 @@ FAIL: int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, int64_t groupId) { - char* ctbName = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); - if (ctbName == NULL) { - return -1; - } - - if (pDataBlock->info.parTbName[0]) { - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); - } else { - char* ctbShortName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, ctbShortName); - taosMemoryFree(ctbShortName); - } - + uint32_t hashValue = 0; SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + if (pTask->pNameMap == NULL) { + pTask->pNameMap = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + } - /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ - SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo; - uint32_t hashValue = - taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); - taosMemoryFree(ctbName); + void* pVal = tSimpleHashGet(pTask->pNameMap, &groupId, sizeof(int64_t)); + if (pVal) { + SBlockName* pBln = (SBlockName*)pVal; + hashValue = pBln->hashValue; + if (!pDataBlock->info.parTbName[0]) { + memcpy(pDataBlock->info.parTbName, pBln->parTbName, strlen(pBln->parTbName)); + } + } else { + char* ctbName = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + if (ctbName == NULL) { + return -1; + } + + if (pDataBlock->info.parTbName[0]) { + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + } else { + buildCtbNameByGroupIdImpl(pTask->shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + } + + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + + /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ + SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo; + hashValue = + taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); + taosMemoryFree(ctbName); + SBlockName bln = {0}; + bln.hashValue = hashValue; + memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); + if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { + tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + } + } bool found = false; // TODO: optimize search diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index a0caffd41f..284d1ecab6 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -224,5 +224,9 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMemoryFree((void*)pTask->id.idStr); } + if (pTask->pNameMap) { + tSimpleHashCleanup(pTask->pNameMap); + } + taosMemoryFree(pTask); } From 09809f2fc099e71cb314e71869aba9f3f6e6358a Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Wed, 7 Jun 2023 11:42:43 +0800 Subject: [PATCH 053/122] docs: go connector version support --- docs/en/14-reference/03-connector/05-go.mdx | 2 +- docs/zh/08-connector/20-go.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index 7e5023b6a7..06d643c6c8 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -29,7 +29,7 @@ REST connections are supported on all platforms that can run Go. ## Version support -Please refer to [version support list](/reference/connector#version-support) +Please refer to [version support list](https://github.com/taosdata/driver-go#remind) ## Supported features diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 5461328182..d431be35cb 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -30,7 +30,7 @@ REST 连接支持所有能运行 Go 的平台。 ## 版本支持 -请参考[版本支持列表](../#版本支持) +请参考[版本支持列表](https://github.com/taosdata/driver-go#remind) ## 支持的功能特性 From 3f26d67cae7442595c6406c1ce5c6b9b1d391d29 Mon Sep 17 00:00:00 2001 From: huolibo Date: Wed, 7 Jun 2023 13:50:32 +0800 Subject: [PATCH 054/122] fix: change kafka doc, delete confluent related content --- docs/en/20-third-party/11-kafka.md | 290 +++++++++++------------------ docs/zh/20-third-party/11-kafka.md | 289 +++++++++++----------------- 2 files changed, 212 insertions(+), 367 deletions(-) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index f09ebb274c..1fc2b57a13 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -16,165 +16,79 @@ TDengine Source Connector is used to read data from TDengine in real-time and se ![TDengine Database Kafka Connector -- streaming integration with kafka connect](kafka/streaming-integration-with-kafka-connect.webp) -## What is Confluent? - -[Confluent](https://www.confluent.io/) adds many extensions to Kafka. include: - -1. Schema Registry -2. REST Proxy -3. Non-Java Clients -4. Many packaged Kafka Connect plugins -5. GUI for managing and monitoring Kafka - Confluent Control Center - -Some of these extensions are available in the community version of Confluent. Some are only available in the enterprise version. -![TDengine Database Kafka Connector -- Confluent platform](kafka/confluentPlatform.webp) - -Confluent Enterprise Edition provides the `confluent` command-line tool to manage various components. - ## Prerequisites 1. Linux operating system 2. Java 8 and Maven installed -3. Git is installed +3. Git/curl/vi is installed 4. TDengine is installed and started. If not, please refer to [Installation and Uninstallation](/operation/pkg-install) -## Install Confluent - -Confluent provides two installation methods: Docker and binary packages. This article only introduces binary package installation. +## Install Kafka Execute in any directory: ```` -curl -O http://packages.confluent.io/archive/7.1/confluent-7.1.1.tar.gz -tar xzf confluent-7.1.1.tar.gz -C /opt/ +curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz +tar xzf kafka_2.13-3.4.0.tgz -C /opt/ +ln -s /opt/kafka_2.13-3.4.0 /opt/kafka ```` -Then you need to add the `$CONFLUENT_HOME/bin` directory to the PATH. +Then you need to add the `$KAFKA_HOME/bin` directory to the PATH. ```title=".profile" -export CONFLUENT_HOME=/opt/confluent-7.1.1 -export PATH=$CONFLUENT_HOME/bin:$PATH +export KAFKA_HOME=/opt/kafka +export PATH=$PATH:$KAFKA_HOME/bin ``` Users can append the above script to the current user's profile file (~/.profile or ~/.bash_profile) -After the installation is complete, you can enter `confluent version` for simple verification: - -``` -# confluent version -confluent - Confluent CLI - -Version: v2.6.1 -Git Ref: 6d920590 -Build Date: 2022-02-18T06:14:21Z -Go Version: go1.17.6 (linux/amd64) -Development: false -``` - ## Install TDengine Connector plugin ### Install from source code -``` +```shell git clone --branch 3.0 https://github.com/taosdata/kafka-connect-tdengine.git cd kafka-connect-tdengine -mvn clean package -unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip +mvn clean package -Dmaven.test.skip=true +unzip -d $KAFKA_HOME/components/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip ``` -The above script first clones the project source code and then compiles and packages it with Maven. After the package is complete, the zip package of the plugin is generated in the `target/components/packages/` directory. Unzip this zip package to plugin path. We used `$CONFLUENT_HOME/share/java/` above because it's a build in plugin path. +The above script first clones the project source code and then compiles and packages it with Maven. After the package is complete, the zip package of the plugin is generated in the `target/components/packages/` directory. Unzip this zip package to plugin path. We used `$KAFKA_HOME/components/` above because it's a build in plugin path. -### Install with confluent-hub +### Add configuration file -[Confluent Hub](https://www.confluent.io/hub) provides a service to download Kafka Connect plugins. After TDengine Kafka Connector is published to Confluent Hub, it can be installed using the command tool `confluent-hub`. -**TDengine Kafka Connector is currently not officially released and cannot be installed in this way**. +add kafka-connect-tdengine plugin path to `plugin.path` in `$KAFKA_HOME/config/connect-distributed.properties`. -## Start Confluent - -``` -confluent local services start +```properties +plugin.path=/usr/share/java,/opt/kafka/components ``` -:::note -Be sure to install the plugin before starting Confluent. Otherwise, Kafka Connect will fail to discover the plugins. -::: +## Start Kafka Services -:::tip -If a component fails to start, try clearing the data and restarting. The data directory will be printed to the console at startup, e.g.: +Use command bellow to start all services: -```title="Console output log" {1} -Using CONFLUENT_CURRENT: /tmp/confluent.106668 -Starting ZooKeeper -ZooKeeper is [UP] -Starting Kafka -Kafka is [UP] -Starting Schema Registry -Schema Registry is [UP] -Starting Kafka REST -Kafka REST is [UP] -Starting Connect -Connect is [UP] -Starting ksqlDB Server -ksqlDB Server is [UP] -Starting Control Center -Control Center is [UP] -``` +```shell +zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties -To clear data, execute `rm -rf /tmp/confluent.106668`. -::: +kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties -### Check Confluent Services Status +connect-distributed.sh -daemon $KAFKA_HOME/config/connect-distributed.properties -Use command bellow to check the status of all service: - -``` -confluent local services status -``` - -The expected output is: -``` -Connect is [UP] -Control Center is [UP] -Kafka is [UP] -Kafka REST is [UP] -ksqlDB Server is [UP] -Schema Registry is [UP] -ZooKeeper is [UP] ``` ### Check Successfully Loaded Plugin After Kafka Connect was completely started, you can use bellow command to check if our plugins are installed successfully: -``` -confluent local services connect plugin list + +```shell +curl http://localhost:8083/connectors ``` -The output should contains `TDengineSinkConnector` and `TDengineSourceConnector` as bellow: +The output as bellow: +```txt +[] ``` -Available Connect Plugins: -[ - { - "class": "com.taosdata.kafka.connect.sink.TDengineSinkConnector", - "type": "sink", - "version": "1.0.0" - }, - { - "class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", - "type": "source", - "version": "1.0.0" - }, -...... -``` - -If not, please check the log file of Kafka Connect. To view the log file path, please execute: - -``` -echo `cat /tmp/confluent.current`/connect/connect.stdout -``` -It should produce a path like:`/tmp/confluent.104086/connect/connect.stdout` - -Besides log file `connect.stdout` there is a file named `connect.properties`. At the end of this file you can see the effective `plugin.path` which is a series of paths joined by comma. If Kafka Connect not found our plugins, it's probably because the installed path is not included in `plugin.path`. ## The use of TDengine Sink Connector @@ -184,40 +98,47 @@ TDengine Sink Connector internally uses TDengine [modeless write interface](/ref The following example synchronizes the data of the topic meters to the target database power. The data format is the InfluxDB Line protocol format. -### Add configuration file +### Add Sink Connector configuration file -``` +```shell mkdir ~/test cd ~/test -vi sink-demo.properties +vi sink-demo.json ``` -sink-demo.properties' content is following: +sink-demo.json' content is following: -```ini title="sink-demo.properties" -name=TDengineSinkConnector -connector.class=com.taosdata.kafka.connect.sink.TDengineSinkConnector -tasks.max=1 -topics=meters -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.user=root -connection.password=taosdata -connection.database=power -db.schemaless=line -data.precision=ns -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +```json title="sink-demo.json" +{ + "name": "TDengineSinkConnector", + "config": { + "connector.class":"com.taosdata.kafka.connect.sink.TDengineSinkConnector", + "tasks.max": "1", + "topics": "meters", + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.user": "root", + "connection.password": "taosdata", + "connection.database": "power", + "db.schemaless": "line", + "data.precision": "ns", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": 1 + } +} ``` Key configuration instructions: -1. `topics=meters` and `connection.database=power` means to subscribe to the data of the topic meters and write to the database power. -2. `db.schemaless=line` means the data in the InfluxDB Line protocol format. +1. `"topics": "meters"` and `"connection.database": "power"` means to subscribe to the data of the topic meters and write to the database power. +2. `"db.schemaless": "line"` means the data in the InfluxDB Line protocol format. -### Create Connector instance +### Create Sink Connector instance -```` -confluent local services connect connector load TDengineSinkConnector --config ./sink-demo.properties +````shell +curl -X POST -d @sink-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" ```` If the above command is executed successfully, the output is as follows: @@ -237,7 +158,10 @@ If the above command is executed successfully, the output is as follows: "tasks.max": "1", "topics": "meters", "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "name": "TDengineSinkConnector" + "name": "TDengineSinkConnector", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": "1", }, "tasks": [], "type": "sink" @@ -258,7 +182,7 @@ meters,location=California.LoSangeles,groupid=3 current=11.3,voltage=221,phase=0 Use kafka-console-producer to write test data to the topic `meters`. ``` -cat test-data.txt | kafka-console-producer --broker-list localhost:9092 --topic meters +cat test-data.txt | kafka-console-producer.sh --broker-list localhost:9092 --topic meters ``` :::note @@ -269,12 +193,12 @@ TDengine Sink Connector will automatically create the database if the target dat Use the TDengine CLI to verify that the sync was successful. -``` +```sql taos> use power; Database changed. taos> select * from meters; - ts | current | voltage | phase | groupid | location | + _ts | current | voltage | phase | groupid | location | =============================================================================================================================================================== 2022-03-28 09:56:51.249000000 | 11.800000000 | 221.000000000 | 0.280000000 | 2 | California.LosAngeles | 2022-03-28 09:56:51.250000000 | 13.400000000 | 223.000000000 | 0.290000000 | 2 | California.LosAngeles | @@ -293,29 +217,34 @@ TDengine Source Connector will convert the data in TDengine data table into [Inf The following sample program synchronizes the data in the database test to the topic tdengine-source-test. -### Add configuration file +### Add Source Connector configuration file -``` -vi source-demo.properties +```shell +vi source-demo.json ``` Input following content: -```ini title="source-demo.properties" -name=TDengineSourceConnector -connector.class=com.taosdata.kafka.connect.source.TDengineSourceConnector -tasks.max=1 -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.username=root -connection.password=taosdata -connection.database=test -connection.attempts=3 -connection.backoff.ms=5000 -topic.prefix=tdengine-source- -poll.interval.ms=1000 -fetch.max.rows=100 -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +```json title="source-demo.json" +{ + "name":"TDengineSourceConnector", + "config":{ + "connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", + "tasks.max": 1, + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.username": "root", + "connection.password": "taosdata", + "connection.database": "test", + "connection.attempts": 3, + "connection.backoff.ms": 5000, + "topic.prefix": "tdengine-source", + "poll.interval.ms": 1000, + "fetch.max.rows": 100, + "topic.per.stable": true, + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter" + } +} ``` ### Prepare test data @@ -340,40 +269,40 @@ INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-1 Use TDengine CLI to execute SQL script -``` +```shell taos -f prepare-source-data.sql ``` ### Create Connector instance -```` -confluent local services connect connector load TDengineSourceConnector --config source-demo.properties -```` +```shell +curl -X POST -d @source-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" +``` ### View topic data Use the kafka-console-consumer command-line tool to monitor data in the topic tdengine-source-test. In the beginning, all historical data will be output. After inserting two new data into TDengine, kafka-console-consumer immediately outputs the two new data. The output is in InfluxDB line protocol format. -```` -kafka-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test +````shell +kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test-meters ```` output: -```` +```txt ...... meters,location="California.SanFrancisco",groupid=2i32 current=10.3f32,voltage=219i32,phase=0.31f32 1538548685000000000 meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=218i32,phase=0.33f32 1538548695000000000 ...... -```` +``` All historical data is displayed. Switch to the TDengine CLI and insert two new pieces of data: -```` +```sql USE test; INSERT INTO d1001 VALUES (now, 13.3, 229, 0.38); INSERT INTO d1002 VALUES (now, 16.3, 233, 0.22); -```` +``` Switch back to kafka-console-consumer, and the command line window has printed out the two pieces of data just inserted. @@ -383,16 +312,16 @@ After testing, use the unload command to stop the loaded connector. View currently active connectors: -```` -confluent local services connect connector status -```` +```shell +curl http://localhost:8083/connectors +``` You should now have two active connectors if you followed the previous steps. Use the following command to unload: -```` -confluent local services connect connector unload TDengineSinkConnector -confluent local services connect connector unload TDengineSourceConnector -```` +```shell +curl -X DELETE http://localhost:8083/connectors/TDengineSinkConnector +curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector +``` ## Configuration reference @@ -430,19 +359,14 @@ The following configuration items apply to TDengine Sink Connector and TDengine 6. `query.interval.ms`: The time range of reading data from TDengine each time, its unit is millisecond. It should be adjusted according to the data flow in rate, the default value is 1000. 7. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is `--`; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is `-`. - - ## Other notes -1. To install plugin to a customized location, refer to https://docs.confluent.io/home/connect/self-managed/install.html#install-connector-manually. -2. To use Kafka Connect without confluent, refer to https://kafka.apache.org/documentation/#connect. +1. To use Kafka Connect, refer to . ## Feedback -https://github.com/taosdata/kafka-connect-tdengine/issues + ## Reference -1. https://www.confluent.io/what-is-apache-kafka -2. https://developer.confluent.io/learn-kafka/kafka-connect/intro -3. https://docs.confluent.io/platform/current/platform.html +1. For more information, see diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 97e78c2fde..641e2d5174 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -16,169 +16,78 @@ TDengine Source Connector 用于把数据实时地从 TDengine 读出来发送 ![TDengine Database Kafka Connector -- streaming integration with kafka connect](kafka/streaming-integration-with-kafka-connect.webp) -## 什么是 Confluent？ - -[Confluent](https://www.confluent.io/) 在 Kafka 的基础上增加很多扩展功能。包括： - -1. Schema Registry -2. REST 代理 -3. 非 Java 客户端 -4. 很多打包好的 Kafka Connect 插件 -5. 管理和监控 Kafka 的 GUI —— Confluent 控制中心 - -这些扩展功能有的包含在社区版本的 Confluent 中，有的只有企业版能用。 -![TDengine Database Kafka Connector -- Confluent introduction](kafka/confluentPlatform.webp) - -Confluent 企业版提供了 `confluent` 命令行工具管理各个组件。 - ## 前置条件 运行本教程中示例的前提条件。 1. Linux 操作系统 2. 已安装 Java 8 和 Maven -3. 已安装 Git +3. 已安装 Git、curl、vi 4. 已安装并启动 TDengine。如果还没有可参考[安装和卸载](/operation/pkg-install) -## 安装 Confluent - -Confluent 提供了 Docker 和二进制包两种安装方式。本文仅介绍二进制包方式安装。 +## 安装 Kafka 在任意目录下执行： -``` -curl -O http://packages.confluent.io/archive/7.1/confluent-7.1.1.tar.gz -tar xzf confluent-7.1.1.tar.gz -C /opt/ +```shell +curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz +tar xzf kafka_2.13-3.4.0.tgz -C /opt/ +ln -s /opt/kafka_2.13-3.4.0 /opt/kafka ``` -然后需要把 `$CONFLUENT_HOME/bin` 目录加入 PATH。 +然后需要把 `$KAFKA_HOME/bin` 目录加入 PATH。 ```title=".profile" -export CONFLUENT_HOME=/opt/confluent-7.1.1 -export PATH=$CONFLUENT_HOME/bin:$PATH +export KAFKA_HOME=/opt/kafka +export PATH=$PATH:$KAFKA_HOME/bin ``` 以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile） -安装完成之后，可以输入`confluent version`做简单验证： - -``` -# confluent version -confluent - Confluent CLI - -Version: v2.6.1 -Git Ref: 6d920590 -Build Date: 2022-02-18T06:14:21Z -Go Version: go1.17.6 (linux/amd64) -Development: false -``` - ## 安装 TDengine Connector 插件 -### 从源码安装 +### 编译插件 -``` +```shell git clone --branch 3.0 https://github.com/taosdata/kafka-connect-tdengine.git cd kafka-connect-tdengine -mvn clean package -unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip +mvn clean package -Dmaven.test.skip=true +unzip -d $KAFKA_HOME/components/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip ``` -以上脚本先 clone 项目源码，然后用 Maven 编译打包。打包完成后在 `target/components/packages/` 目录生成了插件的 zip 包。把这个 zip 包解压到安装插件的路径即可。上面的示例中使用了内置的插件安装路径： `$CONFLUENT_HOME/share/java/`。 +以上脚本先 clone 项目源码，然后用 Maven 编译打包。打包完成后在 `target/components/packages/` 目录生成了插件的 zip 包。把这个 zip 包解压到安装插件的路径即可。上面的示例中使用了内置的插件安装路径： `$KAFKA_HOME/components/`。 -### 用 confluent-hub 安装 +### 配置插件 -[Confluent Hub](https://www.confluent.io/hub) 提供下载 Kafka Connect 插件的服务。在 TDengine Kafka Connector 发布到 Confluent Hub 后可以使用命令工具 `confluent-hub` 安装。 -**TDengine Kafka Connector 目前没有正式发布，不能用这种方式安装**。 +将 kafka-connect-tdengine 插件加入 `$KAFKA_HOME/config/connect-distributed.properties` 配置文件 plugin.path 中 -## 启动 Confluent - -``` -confluent local services start +```properties +plugin.path=/usr/share/java,/opt/kafka/components ``` -:::note -一定要先安装插件再启动 Confluent, 否则加载插件会失败。 -::: +## 启动 Kafka -:::tip -若某组件启动失败，可尝试清空数据，重新启动。数据目录在启动时将被打印到控制台，比如 ： +```shell +zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties -```title="控制台输出日志" {1} -Using CONFLUENT_CURRENT: /tmp/confluent.106668 -Starting ZooKeeper -ZooKeeper is [UP] -Starting Kafka -Kafka is [UP] -Starting Schema Registry -Schema Registry is [UP] -Starting Kafka REST -Kafka REST is [UP] -Starting Connect -Connect is [UP] -Starting ksqlDB Server -ksqlDB Server is [UP] -Starting Control Center -Control Center is [UP] +kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties + +connect-distributed.sh -daemon $KAFKA_HOME/config/connect-distributed.properties ``` -清空数据可执行 `rm -rf /tmp/confluent.106668`。 -::: - -### 验证各个组件是否启动成功 +### 验证 kafka Connect 是否启动成功 输入命令： -``` -confluent local services status +```shell +curl http://localhost:8083/connectors ``` 如果各组件都启动成功，会得到如下输出： +```txt +[] ``` -Connect is [UP] -Control Center is [UP] -Kafka is [UP] -Kafka REST is [UP] -ksqlDB Server is [UP] -Schema Registry is [UP] -ZooKeeper is [UP] -``` - -### 验证插件是否安装成功 - -在 Kafka Connect 组件完全启动后，可用以下命令列出成功加载的插件： - -``` -confluent local services connect plugin list -``` - -如果成功安装，会输出如下： - -```txt {4,9} -Available Connect Plugins: -[ - { - "class": "com.taosdata.kafka.connect.sink.TDengineSinkConnector", - "type": "sink", - "version": "1.0.0" - }, - { - "class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", - "type": "source", - "version": "1.0.0" - }, -...... -``` - -如果插件安装失败，请检查 Kafka Connect 的启动日志是否有异常信息，用以下命令输出日志路径： -``` -echo `cat /tmp/confluent.current`/connect/connect.stdout -``` -该命令的输出类似： `/tmp/confluent.104086/connect/connect.stdout`。 - -与日志文件 `connect.stdout` 同一目录，还有一个文件名为： `connect.properties`。在这个文件的末尾，可以看到最终生效的 `plugin.path`， 它是一系列用逗号分割的路径。如果插件安装失败，很可能是因为实际的安装路径不包含在 `plugin.path` 中。 - ## TDengine Sink Connector 的使用 @@ -188,40 +97,47 @@ TDengine Sink Connector 内部使用 TDengine [无模式写入接口](../../conn 下面的示例将主题 meters 的数据，同步到目标数据库 power。数据格式为 InfluxDB Line 协议格式。 -### 添加配置文件 +### 添加 Sink Connector 配置文件 -``` +```shell mkdir ~/test cd ~/test -vi sink-demo.properties +vi sink-demo.json ``` -sink-demo.properties 内容如下： +sink-demo.json 内容如下： -```ini title="sink-demo.properties" -name=TDengineSinkConnector -connector.class=com.taosdata.kafka.connect.sink.TDengineSinkConnector -tasks.max=1 -topics=meters -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.user=root -connection.password=taosdata -connection.database=power -db.schemaless=line -data.precision=ns -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +```json title="sink-demo.json" +{ + "name": "TDengineSinkConnector", + "config": { + "connector.class":"com.taosdata.kafka.connect.sink.TDengineSinkConnector", + "tasks.max": "1", + "topics": "meters", + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.user": "root", + "connection.password": "taosdata", + "connection.database": "power", + "db.schemaless": "line", + "data.precision": "ns", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": 1 + } +} ``` 关键配置说明： -1. `topics=meters` 和 `connection.database=power`, 表示订阅主题 meters 的数据，并写入数据库 power。 -2. `db.schemaless=line`, 表示使用 InfluxDB Line 协议格式的数据。 +1. `"topics": "meters"` 和 `"connection.database": "power"`, 表示订阅主题 meters 的数据，并写入数据库 power。 +2. `"db.schemaless": "line"`, 表示使用 InfluxDB Line 协议格式的数据。 -### 创建 Connector 实例 +### 创建 Sink Connector 实例 -``` -confluent local services connect connector load TDengineSinkConnector --config ./sink-demo.properties +```shell +curl -X POST -d @sink-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" ``` 若以上命令执行成功，则有如下输出： @@ -241,7 +157,10 @@ confluent local services connect connector load TDengineSinkConnector --config . "tasks.max": "1", "topics": "meters", "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "name": "TDengineSinkConnector" + "name": "TDengineSinkConnector", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": "1", }, "tasks": [], "type": "sink" @@ -261,8 +180,8 @@ meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0 使用 kafka-console-producer 向主题 meters 添加测试数据。 -``` -cat test-data.txt | kafka-console-producer --broker-list localhost:9092 --topic meters +```shell +cat test-data.txt | kafka-console-producer.sh --broker-list localhost:9092 --topic meters ``` :::note @@ -273,12 +192,12 @@ cat test-data.txt | kafka-console-producer --broker-list localhost:9092 --topic 使用 TDengine CLI 验证同步是否成功。 -``` +```sql taos> use power; Database changed. taos> select * from meters; - ts | current | voltage | phase | groupid | location | + _ts | current | voltage | phase | groupid | location | =============================================================================================================================================================== 2022-03-28 09:56:51.249000000 | 11.800000000 | 221.000000000 | 0.280000000 | 2 | California.LosAngeles | 2022-03-28 09:56:51.250000000 | 13.400000000 | 223.000000000 | 0.290000000 | 2 | California.LosAngeles | @@ -297,29 +216,34 @@ TDengine Source Connector 会将 TDengine 数据表中的数据转换成 [Influx 下面的示例程序同步数据库 test 中的数据到主题 tdengine-source-test。 -### 添加配置文件 +### 添加 Source Connector 配置文件 -``` -vi source-demo.properties +```shell +vi source-demo.json ``` 输入以下内容： -```ini title="source-demo.properties" -name=TDengineSourceConnector -connector.class=com.taosdata.kafka.connect.source.TDengineSourceConnector -tasks.max=1 -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.username=root -connection.password=taosdata -connection.database=test -connection.attempts=3 -connection.backoff.ms=5000 -topic.prefix=tdengine-source- -poll.interval.ms=1000 -fetch.max.rows=100 -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +```json title="source-demo.json" +{ + "name":"TDengineSourceConnector", + "config":{ + "connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", + "tasks.max": 1, + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.username": "root", + "connection.password": "taosdata", + "connection.database": "test", + "connection.attempts": 3, + "connection.backoff.ms": 5000, + "topic.prefix": "tdengine-source", + "poll.interval.ms": 1000, + "fetch.max.rows": 100, + "topic.per.stable": true, + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter" + } +} ``` ### 准备测试数据 @@ -344,27 +268,27 @@ INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-1 使用 TDengine CLI, 执行 SQL 文件。 -``` +```shell taos -f prepare-source-data.sql ``` -### 创建 Connector 实例 +### 创建 Source Connector 实例 -``` -confluent local services connect connector load TDengineSourceConnector --config source-demo.properties +```shell +curl -X POST -d @source-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" ``` ### 查看 topic 数据 使用 kafka-console-consumer 命令行工具监控主题 tdengine-source-test 中的数据。一开始会输出所有历史数据， 往 TDengine 插入两条新的数据之后，kafka-console-consumer 也立即输出了新增的两条数据。 输出数据 InfluxDB line protocol 的格式。 -``` -kafka-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test +```shell +kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test-meters ``` 输出： -``` +```txt ...... meters,location="California.SanFrancisco",groupid=2i32 current=10.3f32,voltage=219i32,phase=0.31f32 1538548685000000000 meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=218i32,phase=0.33f32 1538548695000000000 @@ -373,7 +297,7 @@ meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=2 此时会显示所有历史数据。切换到 TDengine CLI， 插入两条新的数据： -``` +```sql USE test; INSERT INTO d1001 VALUES (now, 13.3, 229, 0.38); INSERT INTO d1002 VALUES (now, 16.3, 233, 0.22); @@ -387,15 +311,15 @@ INSERT INTO d1002 VALUES (now, 16.3, 233, 0.22); 查看当前活跃的 connector： -``` -confluent local services connect connector status +```shell +curl http://localhost:8083/connectors ``` 如果按照前述操作，此时应有两个活跃的 connector。使用下面的命令 unload： -``` -confluent local services connect connector unload TDengineSinkConnector -confluent local services connect connector unload TDengineSourceConnector +```shell +curl -X DELETE http://localhost:8083/connectors/TDengineSinkConnector +curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector ``` ## 配置参考 @@ -442,15 +366,12 @@ confluent local services connect connector unload TDengineSourceConnector ## 其他说明 -1. 插件的安装位置可以自定义，请参考官方文档：https://docs.confluent.io/home/connect/self-managed/install.html#install-connector-manually。 -2. 本教程的示例程序使用了 Confluent 平台，但是 TDengine Kafka Connector 本身同样适用于独立安装的 Kafka, 且配置方法相同。关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档： https://kafka.apache.org/documentation/#connect。 +1. 关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档：。 ## 问题反馈 -无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈： https://github.com/taosdata/kafka-connect-tdengine/issues。 +无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈：。 ## 参考 -1. https://www.confluent.io/what-is-apache-kafka -2. https://developer.confluent.io/learn-kafka/kafka-connect/intro -3. https://docs.confluent.io/platform/current/platform.html +1. From 6188f26d57c00d5b85c3a3af8c577d8205820462 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Wed, 7 Jun 2023 14:14:33 +0800 Subject: [PATCH 055/122] docs: add version history --- docs/en/14-reference/03-connector/06-rust.mdx | 9 +++++++-- docs/zh/08-connector/26-rust.mdx | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index fb7c368df1..4a4ef5ba9c 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -27,9 +27,14 @@ The source code for the Rust connectors is located on [GitHub](https://github.co Native connections are supported on the same platforms as the TDengine client driver. Websocket connections are supported on all platforms that can run Go. -## Version support +## Version history -Please refer to [version support list](/reference/connector#version-support) +| connector-rust version | TDengine version | major features | +| :----------------: | :--------------: | :--------------------------------------------------: | +| v0.8.8 | 3.0.5.0 | TMQ: get assignments and seek offset. | +| v0.8.0 | 3.0.4.0 | Support schemaless insert. | +| v0.7.6 | 3.0.3.0 | Support req_id in query. | +| v0.6.0 | 3.0.0.0 | Base features. | The Rust Connector is still under rapid development and is not guaranteed to be backward compatible before 1.0. We recommend using TDengine version 3.0 or higher to avoid known issues. diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index e1e94e068f..34e1fbf271 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -26,9 +26,14 @@ import RustQuery from "../07-develop/04-query-data/_rust.mdx" 原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。 Websocket 连接支持所有能运行 Rust 的平台。 -## 版本支持 +## 版本历史 -请参考[版本支持列表](../#版本支持) +| Rust 连接器版本 | TDengine 版本 | 主要功能 | +| :----------------: | :--------------: | :--------------------------------------------------: | +| v0.8.8 | 3.0.5.0 | 消息订阅：获取消费进度及按照指定进度开始消费。 | +| v0.8.0 | 3.0.4.0 | 支持无模式写入。 | +| v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 | +| v0.6.0 | 3.0.0.0 | 基础功能。 | Rust 连接器仍然在快速开发中，1.0 之前无法保证其向后兼容。建议使用 3.0 版本以上的 TDengine，以避免已知问题。 From 3c1b5d3656f4dca671eac9e1a86f724e5fbb3d4d Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Wed, 7 Jun 2023 16:29:11 +0800 Subject: [PATCH 056/122] docs: update description --- docs/en/14-reference/03-connector/06-rust.mdx | 2 +- docs/zh/08-connector/26-rust.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 4a4ef5ba9c..f32e32f2ad 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -31,7 +31,7 @@ Websocket connections are supported on all platforms that can run Go. | connector-rust version | TDengine version | major features | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.8.8 | 3.0.5.0 | TMQ: get assignments and seek offset. | +| v0.8.8 | 3.0.5.0 or later | TMQ: Get consuming progress and seek offset to consume. | | v0.8.0 | 3.0.4.0 | Support schemaless insert. | | v0.7.6 | 3.0.3.0 | Support req_id in query. | | v0.6.0 | 3.0.0.0 | Base features. | diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 34e1fbf271..a02757b14e 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -30,7 +30,7 @@ Websocket 连接支持所有能运行 Rust 的平台。 | Rust 连接器版本 | TDengine 版本 | 主要功能 | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.8.8 | 3.0.5.0 | 消息订阅：获取消费进度及按照指定进度开始消费。 | +| v0.8.8 | 3.0.5.0 or later | 消息订阅：获取消费进度及按照指定进度开始消费。 | | v0.8.0 | 3.0.4.0 | 支持无模式写入。 | | v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 | | v0.6.0 | 3.0.0.0 | 基础功能。 | From 2641dfa0e7335dbe38477dcbd87d8672fd261ab2 Mon Sep 17 00:00:00 2001 From: huolibo Date: Wed, 7 Jun 2023 16:36:42 +0800 Subject: [PATCH 057/122] docs: jdbc add seek doc --- docs/en/14-reference/03-connector/04-java.mdx | 65 ++++---- docs/zh/08-connector/14-java.mdx | 141 ++++++++++-------- 2 files changed, 114 insertions(+), 92 deletions(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index db49e5f395..b820386380 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -32,25 +32,22 @@ TDengine's JDBC driver implementation is as consistent as possible with the rela Native connections are supported on the same platforms as the TDengine client driver. REST connection supports all platforms that can run Java. -## Version support - -Please refer to [version support list](/reference/connector#version-support) - ## Recent update logs -| taos-jdbcdriver version | major changes | -| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | -| 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | -| 3.2.0 | This version has been deprecated | -| 3.1.0 | JDBC REST connection supports subscription over WebSocket | -| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | -| 3.0.0 | Support for TDengine 3.0 | -| 2.0.42 | fix wasNull interface return value in WebSocket connection | -| 2.0.41 | fix decode method of username and password in REST connection | -| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | -| 2.0.38 | JDBC REST connections add bulk pull function | -| 2.0.37 | Support json tags | -| 2.0.36 | Support schemaless writing | +| taos-jdbcdriver version | major changes | TDengine Minimum version | +| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------: | +| 3.2.1 | subscription add seek function | 3.0.5.0 | +| 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 | +| 3.2.0 | This version has been deprecated | - | +| 3.1.0 | JDBC REST connection supports subscription over WebSocket | - | +| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | - | +| 3.0.0 | Support for TDengine 3.0 | 3.0.0.0 | +| 2.0.42 | fix wasNull interface return value in WebSocket connection | - | +| 2.0.41 | fix decode method of username and password in REST connection | - | +| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | - | +| 2.0.38 | JDBC REST connections add bulk pull function | - | +| 2.0.37 | Support json tags | - | +| 2.0.36 | Support schemaless writing | - | **Note**: adding `batchfetch` to the REST connection and setting it to true will enable the WebSocket connection. @@ -102,6 +99,8 @@ For specific error codes, please refer to. | 0x2319 | user is required | The user name information is missing when creating the connection | | 0x231a | password is required | Password information is missing when creating a connection | | 0x231c | httpEntity is null, sql: | Execution exception occurred during the REST connection | +| 0x231d | can't create connection with server within | Increase the connection time by adding the httpConnectTimeout parameter, or check the connection to the taos adapter. | +| 0x231e | failed to complete the task within the specified time | Increase the execution time by adding the messageWaitTimeout parameter, or check the connection to the taos adapter. | | 0x2350 | unknown error | Unknown exception, please return to the developer on github. | | 0x2352 | Unsupported encoding | An unsupported character encoding set is specified under the native Connection. | | 0x2353 | internal error of database, please see taoslog for more details | An error occurs when the prepare statement is executed on the native connection. Check the taos log to locate the fault. | @@ -117,8 +116,8 @@ For specific error codes, please refer to. | 0x2376 | failed to set consumer topic, topic name is empty | During data subscription creation, the subscription topic name is empty. Check that the specified topic name is correct. | | 0x2377 | consumer reference has been destroyed | The subscription data transfer channel has been closed. Please check the connection to TDengine. | | 0x2378 | consumer create error | Failed to create a data subscription. Check the taos log according to the error message to locate the fault. | -| - | can't create connection with server within | Increase the connection time by adding the httpConnectTimeout parameter, or check the connection to the taos adapter. | -| - | failed to complete the task within the specified time | Increase the execution time by adding the messageWaitTimeout parameter, or check the connection to the taos adapter. | +| 0x2379 | seek offset must not be a negative number | The seek interface parameter cannot be negative. Use the correct parameter | +| 0x237a | vGroup not found in result set | subscription is not bound to the VGroup due to the rebalance mechanism | - [TDengine Java Connector](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) @@ -169,7 +168,7 @@ Add following dependency in the `pom.xml` file of your Maven project: com.taosdata.jdbc taos-jdbcdriver - 3.2.1 + 3.2.2 ``` @@ -913,14 +912,15 @@ public class SchemalessWsTest { public static void main(String[] args) throws SQLException { final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata&batchfetch=true"; - Connection connection = DriverManager.getConnection(url); - init(connection); + try(Connection connection = DriverManager.getConnection(url)){ + init(connection); - SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless"); - writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); - writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); - writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); - System.exit(0); + try(SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless")){ + writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); + writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); + writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); + } + } } private static void init(Connection connection) throws SQLException { @@ -991,6 +991,17 @@ while(true) { `poll` obtains one message each time it is run. +#### Assignment subscription Offset + +``` +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; + +void seek(TopicPartition partition, long offset) throws SQLException; +``` + #### Close subscriptions ```java diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 46800226d7..f63350ea04 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -32,25 +32,22 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致 原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。 REST 连接支持所有能运行 Java 的平台。 -## 版本支持 +## 版本历史 -请参考[版本支持列表](../#版本支持) - -## 最近更新记录 - -| taos-jdbcdriver 版本 | 主要变化 | -| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | -| 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | -| 3.2.0 | 存在连接问题，不推荐使用 | -| 3.1.0 | WebSocket 连接支持订阅功能 | -| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | -| 3.0.0 | 支持 TDengine 3.0 | -| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | -| 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | -| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | -| 2.0.38 | JDBC REST 连接增加批量拉取功能 | -| 2.0.37 | 增加对 json tag 支持 | -| 2.0.36 | 增加对 schemaless 写入支持 | +| taos-jdbcdriver 版本 | 主要变化 | TDengine 最低版本 | +| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :---------------: | +| 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 | +| 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 | +| 3.2.0 | 存在连接问题，不推荐使用 | - | +| 3.1.0 | WebSocket 连接支持订阅功能 | - | +| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | - | +| 3.0.0 | 支持 TDengine 3.0 | 3.0.0.0 | +| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | - | +| 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | - | +| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | - | +| 2.0.38 | JDBC REST 连接增加批量拉取功能 | - | +| 2.0.37 | 增加对 json tag 支持 | - | +| 2.0.36 | 增加对 schemaless 写入支持 | - | **注**：REST 连接中增加 `batchfetch` 参数并设置为 true，将开启 WebSocket 连接。 @@ -80,45 +77,47 @@ JDBC 连接器可能报错的错误码包括 4 种： 具体的错误码请参考： -| Error Code | Description | Suggested Actions | -| ---------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| 0x2301 | connection already closed | 连接已经关闭，检查连接情况，或重新创建连接去执行相关指令。 | -| 0x2302 | this operation is NOT supported currently! | 当前使用接口不支持，可以更换其他连接方式。 | -| 0x2303 | invalid variables | 参数不合法，请检查相应接口规范，调整参数类型及大小。 | -| 0x2304 | statement is closed | statement 已经关闭，请检查 statement 是否关闭后再次使用，或是连接是否正常。 | -| 0x2305 | resultSet is closed | resultSet 结果集已经释放，请检查 resultSet 是否释放后再次使用。 | -| 0x2306 | Batch is empty! | prepareStatement 添加参数后再执行 executeBatch。 | -| 0x2307 | Can not issue data manipulation statements with executeQuery() | 更新操作应该使用 executeUpdate()，而不是 executeQuery()。 | -| 0x2308 | Can not issue SELECT via executeUpdate() | 查询操作应该使用 executeQuery()，而不是 executeUpdate()。 | -| 0x230d | parameter index out of range | 参数越界，请检查参数的合理范围。 | -| 0x230e | connection already closed | 连接已经关闭，请检查 Connection 是否关闭后再次使用，或是连接是否正常。 | -| 0x230f | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 | -| 0x2310 | can't register JDBC-JNI driver | 不能注册 JNI 驱动，请检查 url 是否填写正确。 | -| 0x2312 | url is not set | 请检查 REST 连接 url 是否填写正确。 | -| 0x2314 | numeric value out of range | 请检查获取结果集中数值类型是否使用了正确的接口。 | -| 0x2315 | unknown taos type in tdengine | 在 TDengine 数据类型与 JDBC 数据类型转换时，是否指定了正确的 TDengine 数据类型。 | -| 0x2317 | | REST 连接中使用了错误的请求类型。 | -| 0x2318 | | REST 连接中出现了数据传输异常，请检查网络情况并重试。 | -| 0x2319 | user is required | 创建连接时缺少用户名信息 | -| 0x231a | password is required | 创建连接时缺少密码信息 | -| 0x231c | httpEntity is null, sql: | REST 连接中执行出现异常 | -| 0x2350 | unknown error | 未知异常，请在 github 反馈给开发人员。 | -| 0x2352 | Unsupported encoding | 本地连接下指定了不支持的字符编码集 | -| 0x2353 | internal error of database, please see taoslog for more details | 本地连接执行 prepareStatement 时出现错误，请检查 taos log 进行问题定位。 | -| 0x2354 | JNI connection is NULL | 本地连接执行命令时，Connection 已经关闭。请检查与 TDengine 的连接情况。 | -| 0x2355 | JNI result set is NULL | 本地连接获取结果集，结果集异常，请检查连接情况，并重试。 | -| 0x2356 | invalid num of fields | 本地连接获取结果集的 meta 信息不匹配。 | -| 0x2357 | empty sql string | 填写正确的 SQL 进行执行。 | -| 0x2359 | JNI alloc memory failed, please see taoslog for more details | 本地连接分配内存错误，请检查 taos log 进行问题定位。 | -| 0x2371 | consumer properties must not be null! | 创建订阅时参数为空，请填写正确的参数。 | -| 0x2372 | configs contain empty key, failed to set consumer property | 参数 key 中包含空值，请填写正确的参数。 | -| 0x2373 | failed to set consumer property, | 参数 value 中包含空值，请填写正确的参数。 | -| 0x2375 | topic reference has been destroyed | 创建数据订阅过程中，topic 引用被释放。请检查与 TDengine 的连接情况。 | -| 0x2376 | failed to set consumer topic, topic name is empty | 创建数据订阅过程中，订阅 topic 名称为空。请检查指定的 topic 名称是否填写正确。 | -| 0x2377 | consumer reference has been destroyed | 订阅数据传输通道已经关闭，请检查与 TDengine 的连接情况。 | -| 0x2378 | consumer create error | 创建数据订阅失败，请根据错误信息检查 taos log 进行问题定位。 | -| - | can't create connection with server within | 通过增加参数 httpConnectTimeout 增加连接耗时，或是请检查与 taosAdapter 之间的连接情况。 | -| - | failed to complete the task within the specified time | 通过增加参数 messageWaitTimeout 增加执行耗时，或是请检查与 taosAdapter 之间的连接情况。 | +| Error Code | Description | Suggested Actions | +| ---------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| 0x2301 | connection already closed | 连接已经关闭，检查连接情况，或重新创建连接去执行相关指令。 | +| 0x2302 | this operation is NOT supported currently! | 当前使用接口不支持，可以更换其他连接方式。 | +| 0x2303 | invalid variables | 参数不合法，请检查相应接口规范，调整参数类型及大小。 | +| 0x2304 | statement is closed | statement 已经关闭，请检查 statement 是否关闭后再次使用，或是连接是否正常。 | +| 0x2305 | resultSet is closed | resultSet 结果集已经释放，请检查 resultSet 是否释放后再次使用。 | +| 0x2306 | Batch is empty! | prepareStatement 添加参数后再执行 executeBatch。 | +| 0x2307 | Can not issue data manipulation statements with executeQuery() | 更新操作应该使用 executeUpdate()，而不是 executeQuery()。 | +| 0x2308 | Can not issue SELECT via executeUpdate() | 查询操作应该使用 executeQuery()，而不是 executeUpdate()。 | +| 0x230d | parameter index out of range | 参数越界，请检查参数的合理范围。 | +| 0x230e | connection already closed | 连接已经关闭，请检查 Connection 是否关闭后再次使用，或是连接是否正常。 | +| 0x230f | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 | +| 0x2310 | can't register JDBC-JNI driver | 不能注册 JNI 驱动，请检查 url 是否填写正确。 | +| 0x2312 | url is not set | 请检查 REST 连接 url 是否填写正确。 | +| 0x2314 | numeric value out of range | 请检查获取结果集中数值类型是否使用了正确的接口。 | +| 0x2315 | unknown taos type in tdengine | 在 TDengine 数据类型与 JDBC 数据类型转换时，是否指定了正确的 TDengine 数据类型。 | +| 0x2317 | | REST 连接中使用了错误的请求类型。 | +| 0x2318 | | REST 连接中出现了数据传输异常，请检查网络情况并重试。 | +| 0x2319 | user is required | 创建连接时缺少用户名信息 | +| 0x231a | password is required | 创建连接时缺少密码信息 | +| 0x231c | httpEntity is null, sql: | REST 连接中执行出现异常 | +| 0x231d | can't create connection with server within | 通过增加参数 httpConnectTimeout 增加连接耗时，或是请检查与 taosAdapter 之间的连接情况。 | +| 0x231e | failed to complete the task within the specified time | 通过增加参数 messageWaitTimeout 增加执行耗时，或是请检查与 taosAdapter 之间的连接情况。 | +| 0x2350 | unknown error | 未知异常，请在 github 反馈给开发人员。 | +| 0x2352 | Unsupported encoding | 本地连接下指定了不支持的字符编码集 | +| 0x2353 | internal error of database, please see taoslog for more details | 本地连接执行 prepareStatement 时出现错误，请检查 taos log 进行问题定位。 | +| 0x2354 | JNI connection is NULL | 本地连接执行命令时，Connection 已经关闭。请检查与 TDengine 的连接情况。 | +| 0x2355 | JNI result set is NULL | 本地连接获取结果集，结果集异常，请检查连接情况，并重试。 | +| 0x2356 | invalid num of fields | 本地连接获取结果集的 meta 信息不匹配。 | +| 0x2357 | empty sql string | 填写正确的 SQL 进行执行。 | +| 0x2359 | JNI alloc memory failed, please see taoslog for more details | 本地连接分配内存错误，请检查 taos log 进行问题定位。 | +| 0x2371 | consumer properties must not be null! | 创建订阅时参数为空，请填写正确的参数。 | +| 0x2372 | configs contain empty key, failed to set consumer property | 参数 key 中包含空值，请填写正确的参数。 | +| 0x2373 | failed to set consumer property, | 参数 value 中包含空值，请填写正确的参数。 | +| 0x2375 | topic reference has been destroyed | 创建数据订阅过程中，topic 引用被释放。请检查与 TDengine 的连接情况。 | +| 0x2376 | failed to set consumer topic, topic name is empty | 创建数据订阅过程中，订阅 topic 名称为空。请检查指定的 topic 名称是否填写正确。 | +| 0x2377 | consumer reference has been destroyed | 订阅数据传输通道已经关闭，请检查与 TDengine 的连接情况。 | +| 0x2378 | consumer create error | 创建数据订阅失败，请根据错误信息检查 taos log 进行问题定位。 | +| 0x2379 | seek offset must not be a negative number | seek 接口参数不能为负值，请使用正确的参数 | +| 0x237a | vGroup not found in result set | VGroup 没有分配给当前 consumer，由于 Rebalance 机制导致 Consumer 与 VGroup 不是绑定的关系 | - [TDengine Java Connector](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) @@ -169,7 +168,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖： com.taosdata.jdbc taos-jdbcdriver - 3.2.1 + 3.2.2 ``` @@ -916,14 +915,15 @@ public class SchemalessWsTest { public static void main(String[] args) throws SQLException { final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata&batchfetch=true"; - Connection connection = DriverManager.getConnection(url); - init(connection); + try(Connection connection = DriverManager.getConnection(url)){ + init(connection); - SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless"); - writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); - writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); - writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); - System.exit(0); + try(SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless")){ + writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); + writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); + writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); + } + } } private static void init(Connection connection) throws SQLException { @@ -994,6 +994,17 @@ while(true) { `poll` 每次调用获取一个消息。 +#### 指定订阅 Offset + +``` +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; + +void seek(TopicPartition partition, long offset) throws SQLException; +``` + #### 关闭订阅 ```java From 150e9ae2960da9c8cc146034c0688f3217a9966a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 7 Jun 2023 16:55:03 +0800 Subject: [PATCH 058/122] make interp fill multiple col logic same as window fill --- source/libs/parser/src/parTranslater.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f049af6747..5c841deda0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3041,7 +3041,7 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList return code; } -static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList, bool isInterpFill) { +static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) { if (FILL_MODE_VALUE != pFill->mode && FILL_MODE_VALUE_F != pFill->mode) { return TSDB_CODE_SUCCESS; } @@ -3059,12 +3059,10 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL return code; } - if (!isInterpFill) { - ++fillNo; - } + ++fillNo; } } - if (!isInterpFill && fillNo != LIST_LENGTH(pFillValues->pNodeList)) { + if (fillNo != LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } return TSDB_CODE_SUCCESS; @@ -3075,7 +3073,7 @@ static int32_t translateFillValues(STranslateContext* pCxt, SSelectStmt* pSelect NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { return TSDB_CODE_SUCCESS; } - return checkFillValues(pCxt, (SFillNode*)((SIntervalWindowNode*)pSelect->pWindow)->pFill, pSelect->pProjectionList, false); + return checkFillValues(pCxt, (SFillNode*)((SIntervalWindowNode*)pSelect->pWindow)->pFill, pSelect->pProjectionList); } static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { @@ -3525,7 +3523,7 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true); } if (TSDB_CODE_SUCCESS == code) { - code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList, true); + code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList); } return code; From c612d945ec73114f93623f0116c190608605fc45 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 7 Jun 2023 18:27:29 +0800 Subject: [PATCH 059/122] fix fill value bug --- source/libs/executor/src/timesliceoperator.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 3e4055876d..2421343bd7 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -257,7 +257,8 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // output the result - bool hasInterp = true; + int32_t fillColIndex = 0; + bool hasInterp = true; for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; @@ -307,7 +308,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp case TSDB_FILL_SET_VALUE: case TSDB_FILL_SET_VALUE_F: { - SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal; + SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; @@ -342,6 +343,8 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } colDataSetVal(pDst, rows, (char*)&v, false); } + + ++fillColIndex; break; } From 46baeefea73e2c6edf038482a96e373335d7edef Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 7 Jun 2023 18:39:33 +0800 Subject: [PATCH 060/122] fix test cases --- tests/system-test/2-query/interp.py | 190 ++++++++++++++-------------- 1 file changed, 97 insertions(+), 93 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 019b449dfd..eb6aeec95a 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -1809,47 +1809,10 @@ class TDTestCase: tdSql.checkData(60, 1, 60) # # test fill value - tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123)") + tdSql.query(f"select _irowts, interp(c0), _irowts, interp(c1), _irowts from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123, 456)") tdSql.checkRows(61) - tdSql.checkCols(2) - tdSql.checkData(0, 0, 0) # - tdSql.checkData(1, 0, 123) - tdSql.checkData(4, 0, 123) - tdSql.checkData(5, 0, None) # - tdSql.checkData(6, 0, 123) - tdSql.checkData(9, 0, 123) - tdSql.checkData(10, 0, 10) # - tdSql.checkData(11, 0, 123) - tdSql.checkData(14, 0, 123) - tdSql.checkData(15, 0, None) # - tdSql.checkData(16, 0, 123) - tdSql.checkData(19, 0, 123) - tdSql.checkData(20, 0, 20) # - tdSql.checkData(21, 0, 123) - tdSql.checkData(24, 0, 123) - tdSql.checkData(25, 0, None) # - tdSql.checkData(26, 0, 123) - tdSql.checkData(29, 0, 123) - tdSql.checkData(30, 0, 30) # - tdSql.checkData(31, 0, 123) - tdSql.checkData(34, 0, 123) - tdSql.checkData(35, 0, 35) # - tdSql.checkData(36, 0, 123) - tdSql.checkData(39, 0, 123) - tdSql.checkData(40, 0, 40) # - tdSql.checkData(41, 0, 123) - tdSql.checkData(44, 0, 123) - tdSql.checkData(45, 0, None) # - tdSql.checkData(46, 0, 123) - tdSql.checkData(49, 0, 123) - tdSql.checkData(50, 0, 50) # - tdSql.checkData(51, 0, 123) - tdSql.checkData(54, 0, 123) - tdSql.checkData(55, 0, None) # - tdSql.checkData(59, 0, 123) - tdSql.checkData(60, 0, 55) # - - tdSql.checkData(0, 1, None) # + tdSql.checkCols(5) + tdSql.checkData(0, 1, 0) # tdSql.checkData(1, 1, 123) tdSql.checkData(4, 1, 123) tdSql.checkData(5, 1, None) # @@ -1861,7 +1824,7 @@ class TDTestCase: tdSql.checkData(15, 1, None) # tdSql.checkData(16, 1, 123) tdSql.checkData(19, 1, 123) - tdSql.checkData(20, 1, None) # + tdSql.checkData(20, 1, 20) # tdSql.checkData(21, 1, 123) tdSql.checkData(24, 1, 123) tdSql.checkData(25, 1, None) # @@ -1870,64 +1833,64 @@ class TDTestCase: tdSql.checkData(30, 1, 30) # tdSql.checkData(31, 1, 123) tdSql.checkData(34, 1, 123) - tdSql.checkData(35, 1, None) # + tdSql.checkData(35, 1, 35) # tdSql.checkData(36, 1, 123) tdSql.checkData(39, 1, 123) tdSql.checkData(40, 1, 40) # tdSql.checkData(41, 1, 123) tdSql.checkData(44, 1, 123) - tdSql.checkData(45, 1, 45) # + tdSql.checkData(45, 1, None) # tdSql.checkData(46, 1, 123) tdSql.checkData(49, 1, 123) - tdSql.checkData(50, 1, None) # + tdSql.checkData(50, 1, 50) # tdSql.checkData(51, 1, 123) tdSql.checkData(54, 1, 123) tdSql.checkData(55, 1, None) # - tdSql.checkData(56, 1, 123) tdSql.checkData(59, 1, 123) - tdSql.checkData(60, 1, 60) # + tdSql.checkData(60, 1, 55) # - tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123 + 123)") + tdSql.checkData(0, 3, None) # + tdSql.checkData(1, 3, 456) + tdSql.checkData(4, 3, 456) + tdSql.checkData(5, 3, None) # + tdSql.checkData(6, 3, 456) + tdSql.checkData(9, 3, 456) + tdSql.checkData(10, 3, 10) # + tdSql.checkData(11, 3, 456) + tdSql.checkData(14, 3, 456) + tdSql.checkData(15, 3, None) # + tdSql.checkData(16, 3, 456) + tdSql.checkData(19, 3, 456) + tdSql.checkData(20, 3, None) # + tdSql.checkData(21, 3, 456) + tdSql.checkData(24, 3, 456) + tdSql.checkData(25, 3, None) # + tdSql.checkData(26, 3, 456) + tdSql.checkData(29, 3, 456) + tdSql.checkData(30, 3, 30) # + tdSql.checkData(31, 3, 456) + tdSql.checkData(34, 3, 456) + tdSql.checkData(35, 3, None) # + tdSql.checkData(36, 3, 456) + tdSql.checkData(39, 3, 456) + tdSql.checkData(40, 3, 40) # + tdSql.checkData(41, 3, 456) + tdSql.checkData(44, 3, 456) + tdSql.checkData(45, 3, 45) # + tdSql.checkData(46, 3, 456) + tdSql.checkData(49, 3, 456) + tdSql.checkData(50, 3, None) # + tdSql.checkData(51, 3, 456) + tdSql.checkData(54, 3, 456) + tdSql.checkData(55, 3, None) # + tdSql.checkData(56, 3, 456) + tdSql.checkData(59, 3, 456) + tdSql.checkData(60, 3, 60) # + + tdSql.query(f"select _isfilled, interp(c0), _isfilled, interp(c1), _isfilled from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123 + 123, 234 + 234)") tdSql.checkRows(61) - tdSql.checkCols(2) - tdSql.checkData(0, 0, 0) # - tdSql.checkData(1, 0, 246) - tdSql.checkData(4, 0, 246) - tdSql.checkData(5, 0, None) # - tdSql.checkData(6, 0, 246) - tdSql.checkData(9, 0, 246) - tdSql.checkData(10, 0, 10) # - tdSql.checkData(11, 0, 246) - tdSql.checkData(14, 0, 246) - tdSql.checkData(15, 0, None) # - tdSql.checkData(16, 0, 246) - tdSql.checkData(19, 0, 246) - tdSql.checkData(20, 0, 20) # - tdSql.checkData(21, 0, 246) - tdSql.checkData(24, 0, 246) - tdSql.checkData(25, 0, None) # - tdSql.checkData(26, 0, 246) - tdSql.checkData(29, 0, 246) - tdSql.checkData(30, 0, 30) # - tdSql.checkData(31, 0, 246) - tdSql.checkData(34, 0, 246) - tdSql.checkData(35, 0, 35) # - tdSql.checkData(36, 0, 246) - tdSql.checkData(39, 0, 246) - tdSql.checkData(40, 0, 40) # - tdSql.checkData(41, 0, 246) - tdSql.checkData(44, 0, 246) - tdSql.checkData(45, 0, None) # - tdSql.checkData(46, 0, 246) - tdSql.checkData(49, 0, 246) - tdSql.checkData(50, 0, 50) # - tdSql.checkData(51, 0, 246) - tdSql.checkData(54, 0, 246) - tdSql.checkData(55, 0, None) # - tdSql.checkData(59, 0, 246) - tdSql.checkData(60, 0, 55) # - - tdSql.checkData(0, 1, None) # + tdSql.checkCols(5) + tdSql.checkData(0, 1, 0) # tdSql.checkData(1, 1, 246) tdSql.checkData(4, 1, 246) tdSql.checkData(5, 1, None) # @@ -1939,7 +1902,7 @@ class TDTestCase: tdSql.checkData(15, 1, None) # tdSql.checkData(16, 1, 246) tdSql.checkData(19, 1, 246) - tdSql.checkData(20, 1, None) # + tdSql.checkData(20, 1, 20) # tdSql.checkData(21, 1, 246) tdSql.checkData(24, 1, 246) tdSql.checkData(25, 1, None) # @@ -1948,22 +1911,59 @@ class TDTestCase: tdSql.checkData(30, 1, 30) # tdSql.checkData(31, 1, 246) tdSql.checkData(34, 1, 246) - tdSql.checkData(35, 1, None) # + tdSql.checkData(35, 1, 35) # tdSql.checkData(36, 1, 246) tdSql.checkData(39, 1, 246) tdSql.checkData(40, 1, 40) # tdSql.checkData(41, 1, 246) tdSql.checkData(44, 1, 246) - tdSql.checkData(45, 1, 45) # + tdSql.checkData(45, 1, None) # tdSql.checkData(46, 1, 246) tdSql.checkData(49, 1, 246) - tdSql.checkData(50, 1, None) # + tdSql.checkData(50, 1, 50) # tdSql.checkData(51, 1, 246) tdSql.checkData(54, 1, 246) tdSql.checkData(55, 1, None) # - tdSql.checkData(56, 1, 246) tdSql.checkData(59, 1, 246) - tdSql.checkData(60, 1, 60) # + tdSql.checkData(60, 1, 55) # + + tdSql.checkData(0, 3, None) # + tdSql.checkData(1, 3, 468) + tdSql.checkData(4, 3, 468) + tdSql.checkData(5, 3, None) # + tdSql.checkData(6, 3, 468) + tdSql.checkData(9, 3, 468) + tdSql.checkData(10, 3, 10) # + tdSql.checkData(11, 3, 468) + tdSql.checkData(14, 3, 468) + tdSql.checkData(15, 3, None) # + tdSql.checkData(16, 3, 468) + tdSql.checkData(19, 3, 468) + tdSql.checkData(20, 3, None) # + tdSql.checkData(21, 3, 468) + tdSql.checkData(24, 3, 468) + tdSql.checkData(25, 3, None) # + tdSql.checkData(26, 3, 468) + tdSql.checkData(29, 3, 468) + tdSql.checkData(30, 3, 30) # + tdSql.checkData(31, 3, 468) + tdSql.checkData(34, 3, 468) + tdSql.checkData(35, 3, None) # + tdSql.checkData(36, 3, 468) + tdSql.checkData(39, 3, 468) + tdSql.checkData(40, 3, 40) # + tdSql.checkData(41, 3, 468) + tdSql.checkData(44, 3, 468) + tdSql.checkData(45, 3, 45) # + tdSql.checkData(46, 3, 468) + tdSql.checkData(49, 3, 468) + tdSql.checkData(50, 3, None) # + tdSql.checkData(51, 3, 468) + tdSql.checkData(54, 3, 468) + tdSql.checkData(55, 3, None) # + tdSql.checkData(56, 3, 468) + tdSql.checkData(59, 3, 468) + tdSql.checkData(60, 3, 60) # # test fill prev tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(prev)") @@ -2138,7 +2138,7 @@ class TDTestCase: tdSql.checkData(3, i, None) tdSql.checkData(4, i, None) - tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(value, 1)") + tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(value, 1, 1, 1, 1)") tdSql.checkRows(5) tdSql.checkCols(4) @@ -2564,6 +2564,10 @@ class TDTestCase: tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _isfilled = true range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _irowts > 0 range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + # fill value number mismatch + tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(value, 1, 2)") + tdSql.error(f"select interp(c0), interp(c1) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(value, 1)") + From 9b749b8faa0c2abeb622757161c70ec9112d1989 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 8 Jun 2023 02:43:44 +0000 Subject: [PATCH 061/122] change link op t --- contrib/CMakeLists.txt | 2 +- examples/c/CMakeLists.txt | 12 ++++++------ source/dnode/vnode/CMakeLists.txt | 2 +- source/libs/stream/CMakeLists.txt | 2 +- utils/test/c/CMakeLists.txt | 16 ++++++++-------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 59986a3b3c..fdb9f102f0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -274,7 +274,7 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_TOOLS "" OFF) option(WITH_LIBURING "" OFF) IF (TD_LINUX) - option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) ELSE() option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) ENDIF() diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index e14c4e60d9..07fc2fd71b 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -42,27 +42,27 @@ IF (TD_LINUX) ) target_link_libraries(tmq - taos_static + taos ) target_link_libraries(stream_demo - taos_static + taos ) target_link_libraries(schemaless - taos_static + taos ) target_link_libraries(prepare - taos_static + taos ) target_link_libraries(demo - taos_static + taos ) target_link_libraries(asyncdemo - taos_static + taos ) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b18cb8e282..b7bfc57cd5 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -103,7 +103,7 @@ target_link_libraries( # PUBLIC bdb # PUBLIC scalar - PUBLIC rocksdb-shared + PUBLIC rocksdb PUBLIC transport PUBLIC stream PUBLIC index diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index fa6c709c8f..d1ef7fe3c1 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -11,7 +11,7 @@ if(${BUILD_WITH_ROCKSDB}) IF (TD_LINUX) target_link_libraries( stream - PUBLIC rocksdb-shared tdb + PUBLIC rocksdb tdb PRIVATE os util transport qcom executor wal index ) ELSE() diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 87b0d11d1c..71dfd710a5 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -9,35 +9,35 @@ add_executable(get_db_name_test get_db_name_test.c) add_executable(tmq_offset tmqOffset.c) target_link_libraries( tmq_offset - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( create_table - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( tmq_demo - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( tmq_sim - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( tmq_taosx_ci - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os @@ -45,7 +45,7 @@ target_link_libraries( target_link_libraries( write_raw_block_test - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os @@ -53,7 +53,7 @@ target_link_libraries( target_link_libraries( sml_test - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os @@ -61,7 +61,7 @@ target_link_libraries( target_link_libraries( get_db_name_test - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os From 4d574ca611f8ced2c37ba4be33bd87f6f6ea04de Mon Sep 17 00:00:00 2001 From: xleili Date: Wed, 7 Jun 2023 16:57:30 +0800 Subject: [PATCH 062/122] fix: exclude install and remove lbrocksdb.so --- packaging/deb/DEBIAN/preinst | 1 - packaging/deb/DEBIAN/prerm | 1 - packaging/deb/makedeb.sh | 2 -- packaging/rpm/tdengine.spec | 4 ---- packaging/tools/install.sh | 12 ------------ packaging/tools/makepkg.sh | 3 --- packaging/tools/post.sh | 10 ---------- packaging/tools/remove.sh | 2 -- 8 files changed, 35 deletions(-) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index d6558d5b3b..904a946e20 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,5 +80,4 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : -[ -f ${install_main_dir}/driver/librocksdb.* ] && ${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 8f8d472867..0d63115a04 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,7 +40,6 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/librocksdb.* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 024c69deb1..9f49cf345a 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -31,7 +31,6 @@ cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" wslibfile="libtaosws.so" -rocksdblib="librocksdb.so.8" # create install dir install_home_path="/usr/local/taos" @@ -95,7 +94,6 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver -[ -f ${compile_dir}/build/lib/${rocksdblib} ] && cp ${compile_dir}/build/lib/${rocksdblib} ${pkg_dir}${install_home_path}/driver ||: [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 2b056c376a..52d5335003 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -45,7 +45,6 @@ echo buildroot: %{buildroot} libfile="libtaos.so.%{_version}" wslibfile="libtaosws.so" -rocksdblib="librocksdb.so.8" # create install path, and cp file mkdir -p %{buildroot}%{homepath}/bin @@ -93,7 +92,6 @@ if [ -f %{_compiledir}/build/bin/taosadapter ]; then fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: -[ -f %{_compiledir}/build/lib/${rocksdblib} ] && cp %{_compiledir}/build/lib/${rocksdblib} %{buildroot}%{homepath}/driver ||: cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include @@ -176,7 +174,6 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f %{homepath}/driver/libtaos* || : -${csudo}rm -f %{homepath}/driver/librocksdb* || : #Scripts executed after installation %post @@ -222,7 +219,6 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 9aa019f218..1b47b10520 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -250,30 +250,18 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : - ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib_link_dir}/librocksdb.so.8 - ${csudo}ln -sf ${lib_link_dir}/librocksdb.so.8 ${lib_link_dir}/librocksdb.so - - ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib_link_dir}/librocksdb.so.8 - ${csudo}ln -sf ${lib_link_dir}/librocksdb.so.8 ${lib_link_dir}/librocksdb.so - - [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib64_link_dir}/librocksdb.so.8 || : - ${csudo}ln -sf ${lib64_link_dir}/librocksdb.so.8 ${lib64_link_dir}/librocksdb.so || : - [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index ab45c684c4..b0537e8bcf 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -111,11 +111,9 @@ fi if [ "$osType" == "Darwin" ]; then lib_files="${build_dir}/lib/libtaos.${version}.dylib" wslib_files="${build_dir}/lib/libtaosws.dylib" - rocksdb_lib_files="${build_dir}/lib/librocksdb.dylib.8.1.1" else lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so" - rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" @@ -338,7 +336,6 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : -[ -f ${rocksdb_lib_files} ] && cp ${rocksdb_lib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 10de87966f..fc392c9684 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -202,19 +202,10 @@ function install_lib() { log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : - - #rocksdb - [ -f ${lib_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb* || : - [ -f ${lib64_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib64_link_dir}/librocksdb* || : - - #rocksdb - [ -f ${lib_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb* || : - [ -f ${lib64_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib64_link_dir}/librocksdb* || : [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : - ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 @@ -223,7 +214,6 @@ function install_lib() { if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 - ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib64_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} 2>>${install_log_path} fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index a17b29983c..be2c26c309 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -142,11 +142,9 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : From 576e3d1156ae9a529e6a305d7f712e6a04dabea0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Jun 2023 11:49:49 +0800 Subject: [PATCH 063/122] chore: erase duplicated func node for window logic node --- source/libs/parser/src/parTranslater.c | 2 +- source/libs/planner/src/planLogicCreater.c | 19 ++++++++++++++++++ source/libs/planner/src/planOptimizer.c | 23 +++------------------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a9fe65492d..9f94c7cb8c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4940,7 +4940,7 @@ static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); - snprintf(pTable->table.tableAlias, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); + snprintf(pTable->table.tableAlias, sizeof(pTable->table.tableAlias), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); *pOutput = (SNode*)pTable; return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 5bbc9acdad..8c30beffc9 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -749,6 +749,25 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL); } + // erase duplicated funcNode by filtering colNode in pSelect->pProjectionList + int32_t funcIndex = 0; + SNode * pFunc = NULL, *pProject = NULL; + FOREACH(pFunc, pWindow->pFuncs) { + bool exist = false; + FOREACH(pProject, pSelect->pProjectionList) { + if (0 != ((SFunctionNode*)pFunc)->node.aliasName[0] && + 0 == strncmp(((SFunctionNode*)pFunc)->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { + exist = true; + break; + } + } + if (!exist) { + nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, funcIndex)); + } else { + ++funcIndex; + } + } + if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 36378a5414..8b75fe6b33 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1385,18 +1385,7 @@ static SNode* smaIndexOptFindWStartFunc(SNodeList* pSmaFuncs) { return NULL; } -static SNode* smaIndexOptFuncInProject(SNodeList* pProjects, SFunctionNode* pFunc) { - SNode* pProject = NULL; - FOREACH(pProject, pProjects) { - if (0 != pFunc->node.aliasName[0] && - 0 == strncmp(pFunc->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { - return pProject; - } - } - return NULL; -} - -static int32_t smaIndexOptCreateSmaCols(SWindowLogicNode* pWindow, uint64_t tableId, SNodeList* pSmaFuncs, +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput) { SNodeList* pCols = NULL; SNode* pFunc = NULL; @@ -1404,16 +1393,11 @@ static int32_t smaIndexOptCreateSmaCols(SWindowLogicNode* pWindow, uint64_t tabl int32_t index = 0; int32_t smaFuncIndex = -1; bool hasWStart = false; - - SProjectLogicNode* pProject = (SProjectLogicNode*)pWindow->node.pParent; - FOREACH(pFunc, pWindow->pFuncs) { + FOREACH(pFunc, pFuncs) { smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; } else { - if (pProject && !smaIndexOptFuncInProject(pProject->pProjections, (SFunctionNode*)pFunc)) { - continue; - } code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 1)); if (TSDB_CODE_SUCCESS != code) { break; @@ -1460,11 +1444,10 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; } - SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaCols(pWindow, pIndex->dstTbUid, pSmaFuncs, pCols); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols); } nodesDestroyList(pSmaFuncs); return code; From 90b4580ba9ccb784549cec2be8102e72bcabcdee Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Jun 2023 14:16:05 +0800 Subject: [PATCH 064/122] chore: only filter window pseudo column --- source/libs/planner/src/planLogicCreater.c | 37 +++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 8c30beffc9..a253405404 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -749,22 +749,29 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL); } - // erase duplicated funcNode by filtering colNode in pSelect->pProjectionList - int32_t funcIndex = 0; - SNode * pFunc = NULL, *pProject = NULL; - FOREACH(pFunc, pWindow->pFuncs) { - bool exist = false; - FOREACH(pProject, pSelect->pProjectionList) { - if (0 != ((SFunctionNode*)pFunc)->node.aliasName[0] && - 0 == strncmp(((SFunctionNode*)pFunc)->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { - exist = true; - break; + // erase duplicated Window Pseudo funcNode by filtering colNode in pSelect->pProjectionList + if (pSelect->pProjectionList) { + int32_t funcIndex = 0; + SNode * pFunc = NULL, *pProject = NULL; + FOREACH(pFunc, pWindow->pFuncs) { + if (!fmIsWindowPseudoColumnFunc(((SFunctionNode*)pFunc)->funcId)) { + ++funcIndex; + continue; + } + bool exist = false; + FOREACH(pProject, pSelect->pProjectionList) { + if (0 != ((SFunctionNode*)pFunc)->node.aliasName[0] && + 0 == strncmp(((SFunctionNode*)pFunc)->node.aliasName, ((SColumnNode*)pProject)->colName, + TSDB_COL_NAME_LEN)) { + exist = true; + break; + } + } + if (!exist) { + nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, funcIndex)); + } else { + ++funcIndex; } - } - if (!exist) { - nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, funcIndex)); - } else { - ++funcIndex; } } From e56c5950aa34dc31c5f074c1aeadcda9d088204a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Jun 2023 14:29:44 +0800 Subject: [PATCH 065/122] chore: more check --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a253405404..ffabfcc0dd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -760,7 +760,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm } bool exist = false; FOREACH(pProject, pSelect->pProjectionList) { - if (0 != ((SFunctionNode*)pFunc)->node.aliasName[0] && + if (QUERY_NODE_COLUMN == nodeType(pProject) && 0 != ((SFunctionNode*)pFunc)->node.aliasName[0] && 0 == strncmp(((SFunctionNode*)pFunc)->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { exist = true; From c3726b82643cd8eef15bd6941df6a8508f6bd3e7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Jun 2023 14:42:10 +0800 Subject: [PATCH 066/122] chore: more check --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ffabfcc0dd..dd8fd3861d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -750,7 +750,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm } // erase duplicated Window Pseudo funcNode by filtering colNode in pSelect->pProjectionList - if (pSelect->pProjectionList) { + if (WINDOW_TYPE_INTERVAL == pWindow->winType && pSelect->pProjectionList) { int32_t funcIndex = 0; SNode * pFunc = NULL, *pProject = NULL; FOREACH(pFunc, pWindow->pFuncs) { From 3af153bae6aceb961e983a6a98beeb97059fa3bb Mon Sep 17 00:00:00 2001 From: sunpeng Date: Thu, 8 Jun 2023 14:43:07 +0800 Subject: [PATCH 067/122] docs: add docs for assignment and seek --- .../14-reference/03-connector/07-python.mdx | 41 ++++++++++++- .../examples/python/tmq_assignment_example.py | 58 +++++++++++++++++++ .../python/tmq_websocket_assgnment_example.py | 57 ++++++++++++++++++ docs/zh/08-connector/30-python.mdx | 12 ++++ 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 docs/examples/python/tmq_assignment_example.py create mode 100644 docs/examples/python/tmq_websocket_assgnment_example.py diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index b263af8ea6..ba322348cf 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -453,6 +453,44 @@ As the way to connect introduced above but add `req_id` argument. +### Subscription + +Connector support data subscription. For more information about subscroption, please refer to [Data Subscription](../../develop/tmq/). + + + + +The `consumer` in the connector has the subscription api. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/). + +```python +{{#include docs/examples/python/tmq_example.py}} +``` + +There is an `assignment` API in the connector, which can get the assignment of the topic. And there is a `seek` api to reset the assignment of the topic. + +```python +{{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} +``` + + + + + +In addition to native connections, the connector also supports subscriptions via websockets. + +```python +{{#include docs/examples/python/tmq_websocket_example.py}} +``` + +There is an `assignment` API in the connector, which can get the assignment of the topic. And there is a `seek` api to reset the assignment of the topic. + +```python +{{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} +``` + + + + ### Schemaless Insert Connector support schemaless insert. @@ -507,7 +545,8 @@ Insert with req_id argument | Example program links | Example program content | | ------------------------------------------------------------------------------------------------------------- | ------------------- ---- | -| [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, bind multiple rows at once | +| [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, +bind multiple rows at once | | [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | bind_row.py | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB line protocol writing | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | Use JSON type tags | diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py new file mode 100644 index 0000000000..a07347a9b9 --- /dev/null +++ b/docs/examples/python/tmq_assignment_example.py @@ -0,0 +1,58 @@ +import taos +from taos.tmq import Consumer +import taosws + + +def prepare(): + conn = taos.connect() + conn.execute("drop topic if exists tmq_assignment_demo_topic") + conn.execute("drop database if exists tmq_assignment_demo_db") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.select_db("tmq_assignment_demo_db") + conn.execute( + "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") + conn.execute( + "create topic if not exists tmq_assignment_demo_topic as select ts, c1, c2, c3 from tmq_assignment_demo_table") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-2s, 1, 1.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-1s, 2, 2.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now, 3, 3.0, 'tmq test')") + + +def taos_get_assignment_and_seek_demo(): + prepare() + consumer = Consumer( + { + "group.id": "0", + # should disable snapshot, + # otherwise it will cause invalid params error + "experimental.snapshot.enable": "false", + } + ) + consumer.subscribe(["tmq_assignment_demo_topic"]) + + # get topic assignment + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + # poll + consumer.poll(1) + consumer.poll(1) + + # get topic assignment again + after_pool_assignments = consumer.assignment() + for assignment in after_pool_assignments: + print(assignment) + + # seek to the beginning + for assignment in assignments: + consumer.seek(assignment) + + # now the assignment should be the same as before poll + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + +if __name__ == '__main__': + taosws_get_assignment_and_seek_demo() diff --git a/docs/examples/python/tmq_websocket_assgnment_example.py b/docs/examples/python/tmq_websocket_assgnment_example.py new file mode 100644 index 0000000000..0f8e4a2804 --- /dev/null +++ b/docs/examples/python/tmq_websocket_assgnment_example.py @@ -0,0 +1,57 @@ +import taos +import taosws + + +def prepare(): + conn = taos.connect() + conn.execute("drop topic if exists tmq_assignment_demo_topic") + conn.execute("drop database if exists tmq_assignment_demo_db") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.select_db("tmq_assignment_demo_db") + conn.execute( + "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") + conn.execute( + "create topic if not exists tmq_assignment_demo_topic as select ts, c1, c2, c3 from tmq_assignment_demo_table") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-2s, 1, 1.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-1s, 2, 2.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now, 3, 3.0, 'tmq test')") + + +def taosws_get_assignment_and_seek_demo(): + prepare() + consumer = taosws.Consumer(conf={ + "td.connect.websocket.scheme": "ws", + # should disable snapshot, + # otherwise it will cause invalid params error + "experimental.snapshot.enable": "false", + "group.id": "0", + }) + consumer.subscribe(["tmq_assignment_demo_topic"]) + + # get topic assignment + assignments = consumer.assignment() + for assignment in assignments: + print(assignment.to_string()) + + # poll + consumer.poll(1) + consumer.poll(1) + + # get topic assignment again + after_poll_assignments = consumer.assignment() + for assignment in after_poll_assignments: + print(assignment.to_string()) + + # seek to the beginning + for assignment in assignments: + for a in assignment.assignments(): + consumer.seek(assignment.topic(), a.vg_id(), a.offset()) + + # now the assignment should be the same as before poll + assignments = consumer.assignment() + for assignment in assignments: + print(assignment.to_string()) + + +if __name__ == '__main__': + taosws_get_assignment_and_seek_demo() diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 1037d66f17..5c310ff79f 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -467,6 +467,12 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 {{#include docs/examples/python/tmq_example.py}} ``` +连接器提供了 `assignment`接口，用于获取 topic assignment 的功能，可以查询订阅的 assignment 的消费进度，并提供 `seek` 接口，用于重置 topic 的 assignment。 + +```python +{{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} +``` + @@ -477,6 +483,12 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 {{#include docs/examples/python/tmq_websocket_example.py}} ``` +连接器提供了 `assignment` 接口，用于获取 topic assignment 的功能，可以查询订阅的 topic 的消费进度，并提供 `seek` 接口，用于重置 topic 的消费进度。 + +```python +{{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} +``` + From 239c031487c584889cb6d97497265af7d8e5933e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 8 Jun 2023 15:41:57 +0800 Subject: [PATCH 068/122] test: add precision us and ns case --- tests/system-test/1-insert/precisionNS.py | 293 ++++++++++++++++++++++ tests/system-test/1-insert/precisionUS.py | 287 +++++++++++++++++++++ 2 files changed, 580 insertions(+) create mode 100644 tests/system-test/1-insert/precisionNS.py create mode 100644 tests/system-test/1-insert/precisionUS.py diff --git a/tests/system-test/1-insert/precisionNS.py b/tests/system-test/1-insert/precisionNS.py new file mode 100644 index 0000000000..be8f1e21dc --- /dev/null +++ b/tests/system-test/1-insert/precisionNS.py @@ -0,0 +1,293 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000*1000*1000 + self.childCnt = 5 + self.childRow = 10000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 2 precision 'ns' " + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # create stream + sql = "create stream ma into sta as select count(ts) from st interval(100b)" + tdLog.info(sql) + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + + + + # check time macro + def checkTimeMacro(self): + # 2 week + val = 2 + nsval = val*7*24*60*60*1000*1000*1000 + expectVal = self.childCnt * self.childRow + sql = f"select count(ts) from st where timediff(ts - {val}w, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + + # 20 day + val = 20 + nsval = val*24*60*60*1000*1000*1000 + uint = "d" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + + # 30 hour + val = 30 + nsval = val*60*60*1000*1000*1000 + uint = "h" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + + # 90 minutes + val = 90 + nsval = val*60*1000*1000*1000 + uint = "m" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 2s + val = 2 + nsval = val*1000*1000*1000 + uint = "s" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 20a + val = 5 + nsval = val*1000*1000 + uint = "a" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 300u + val = 300 + nsval = val*1000 + uint = "u" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 8b + val = 8 + sql = f"select timediff(ts - {val}b, ts1) from st " + self.checkExpect(sql, val) + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # where + def checkWhere(self): + cnt = 300 + start = self.ts - cnt + sql = f"select count(ts) from st where ts >= {start} and ts <= {self.ts}" + self.checkExpect(sql, cnt) + + for i in range(50): + cnt = random.randint(1,40000) + base = 2000 + start = self.ts - cnt - base + end = self.ts - base + sql = f"select count(ts) from st where ts >= {start} and ts < {end}" + self.checkExpect(sql, cnt) + + # stream + def checkStream(self): + allRows = self.childCnt * self.childRow + # ensure write data is expected + sql = "select count(*) from (select diff(ts) as a from (select ts from st order by ts asc)) where a=1;" + self.checkExpect(sql, allRows - 1) + + # stream count is ok + sql =f"select count(*) from sta" + cnt = int(allRows / 100) - 1 # last window is not close, so need reduce one + self.checkExpect(sql, cnt) + + # check fields + sql =f"select count(*) from sta where `count(ts)` != 100" + self.checkExpect(sql, 0) + + # check timestamp + sql =f"select count(*) from (select diff(`_wstart`) from sta)" + self.checkExpect(sql, cnt - 1) + sql =f"select count(*) from (select diff(`_wstart`) as a from sta) where a != 100" + self.checkExpect(sql, 0) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # time macro like 1w 1d 1h 1m 1s 1a 1u 1b + self.checkTimeMacro() + + # check where + self.checkWhere() + + # check stream + self.checkStream() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/precisionUS.py b/tests/system-test/1-insert/precisionUS.py new file mode 100644 index 0000000000..1b41d66010 --- /dev/null +++ b/tests/system-test/1-insert/precisionUS.py @@ -0,0 +1,287 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000*1000 + self.childCnt = 5 + self.childRow = 10000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 2 precision 'us' " + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # create stream + sql = "create stream ma into sta as select count(ts) from st interval(100u)" + tdLog.info(sql) + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + + # check time macro + def checkTimeMacro(self): + # 2 week + val = 2 + usval = val*7*24*60*60*1000*1000 + expectVal = self.childCnt * self.childRow + sql = f"select count(ts) from st where timediff(ts - {val}w, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # 20 day + val = 20 + usval = val*24*60*60*1000*1000 + uint = "d" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # 30 hour + val = 30 + usval = val*60*60*1000*1000 + uint = "h" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # 90 minutes + val = 90 + usval = val*60*1000*1000 + uint = "m" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + # 2s + val = 2 + usval = val*1000*1000 + uint = "s" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + # 20a + val = 20 + usval = val*1000 + uint = "a" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + # 300u + val = 300 + usval = val*1 + uint = "u" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # where + def checkWhere(self): + cnt = 300 + start = self.ts - cnt + sql = f"select count(ts) from st where ts >= {start} and ts <= {self.ts}" + self.checkExpect(sql, cnt) + + for i in range(50): + cnt = random.randint(1,40000) + base = 2000 + start = self.ts - cnt - base + end = self.ts - base + sql = f"select count(ts) from st where ts >= {start} and ts < {end}" + self.checkExpect(sql, cnt) + + # stream + def checkStream(self): + allRows = self.childCnt * self.childRow + # ensure write data is expected + sql = "select count(*) from (select diff(ts) as a from (select ts from st order by ts asc)) where a=1;" + self.checkExpect(sql, allRows - 1) + + # stream count is ok + sql =f"select count(*) from sta" + cnt = int(allRows / 100) - 1 # last window is not close, so need reduce one + self.checkExpect(sql, cnt) + + # check fields + sql =f"select count(*) from sta where `count(ts)` != 100" + self.checkExpect(sql, 0) + + # check timestamp + sql =f"select count(*) from (select diff(`_wstart`) from sta)" + self.checkExpect(sql, cnt - 1) + sql =f"select count(*) from (select diff(`_wstart`) as a from sta) where a != 100" + self.checkExpect(sql, 0) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # time macro like 1w 1d 1h 1m 1s 1a 1u + self.checkTimeMacro() + + # check where + self.checkWhere() + + # check stream + self.checkStream() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 9eba09df8758e85e27cb5815cf66a4ae29acedef Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 8 Jun 2023 15:47:28 +0800 Subject: [PATCH 069/122] test: add ns and us precision case to task list --- tests/parallel_test/cases.task | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a81209b835..801e6aa1bd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -616,6 +616,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py From e5e724a8baf4f0beed5c561a0ef8d5420a4175a7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Jun 2023 16:13:31 +0800 Subject: [PATCH 070/122] chore: code optimization --- source/libs/planner/src/planLogicCreater.c | 70 ++++++++++++++-------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index dd8fd3861d..5a8512c12d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -732,6 +732,49 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return code; } +static int32_t eraseDuplicatedWindowPseudoCol(SWindowLogicNode* pWindow, SNodeList* pProjections) { + int32_t code = 0; + int32_t funcIndex = 0; + SSHashObj* pHashFunc = NULL; + SNode* pFuncNode = NULL; + FOREACH(pFuncNode, pWindow->pFuncs) { + SFunctionNode* pFunc = (SFunctionNode*)pFuncNode; + if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { + ++funcIndex; + continue; + } + if (!pHashFunc && !(pHashFunc = tSimpleHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + + void* hashVal = tSimpleHashGet(pHashFunc, &pFunc->funcId, sizeof(pFunc->funcId)); + if (!hashVal) { + tSimpleHashPut(pHashFunc, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); + ++funcIndex; + continue; + } + + bool exist = false; + SNode* pProject = NULL; + FOREACH(pProject, pProjections) { + if (QUERY_NODE_COLUMN == nodeType(pProject) && 0 != pFunc->node.aliasName[0] && + 0 == strncmp(pFunc->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { + exist = true; + break; + } + } + if (!exist) { + nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, funcIndex)); + } else { + nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, *(int32_t*)hashVal)); + tSimpleHashPut(pHashFunc, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); + } + } + tSimpleHashCleanup(pHashFunc); + return code; +} + static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { if (pCxt->pPlanCxt->streamQuery) { @@ -749,30 +792,9 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL); } - // erase duplicated Window Pseudo funcNode by filtering colNode in pSelect->pProjectionList - if (WINDOW_TYPE_INTERVAL == pWindow->winType && pSelect->pProjectionList) { - int32_t funcIndex = 0; - SNode * pFunc = NULL, *pProject = NULL; - FOREACH(pFunc, pWindow->pFuncs) { - if (!fmIsWindowPseudoColumnFunc(((SFunctionNode*)pFunc)->funcId)) { - ++funcIndex; - continue; - } - bool exist = false; - FOREACH(pProject, pSelect->pProjectionList) { - if (QUERY_NODE_COLUMN == nodeType(pProject) && 0 != ((SFunctionNode*)pFunc)->node.aliasName[0] && - 0 == strncmp(((SFunctionNode*)pFunc)->node.aliasName, ((SColumnNode*)pProject)->colName, - TSDB_COL_NAME_LEN)) { - exist = true; - break; - } - } - if (!exist) { - nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, funcIndex)); - } else { - ++funcIndex; - } - } + if (TSDB_CODE_SUCCESS == code && WINDOW_TYPE_INTERVAL == pWindow->winType && pSelect->pProjectionList) { + // erase duplicated Window Pseudo funcNode by filtering colNode in pSelect->pProjectionList + code = eraseDuplicatedWindowPseudoCol(pWindow, pSelect->pProjectionList); } if (TSDB_CODE_SUCCESS == code) { From df67a61519c9c84187a4b7ddcbc83165644021ae Mon Sep 17 00:00:00 2001 From: huolibo Date: Thu, 8 Jun 2023 16:40:35 +0800 Subject: [PATCH 071/122] docs(driver): TDengine description --- docs/en/14-reference/03-connector/04-java.mdx | 28 +++++++++---------- docs/zh/08-connector/14-java.mdx | 28 +++++++++---------- .../src/main/resources/application.properties | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index b820386380..9c5a852c70 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -34,20 +34,20 @@ REST connection supports all platforms that can run Java. ## Recent update logs -| taos-jdbcdriver version | major changes | TDengine Minimum version | -| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------: | -| 3.2.1 | subscription add seek function | 3.0.5.0 | -| 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 | -| 3.2.0 | This version has been deprecated | - | -| 3.1.0 | JDBC REST connection supports subscription over WebSocket | - | -| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | - | -| 3.0.0 | Support for TDengine 3.0 | 3.0.0.0 | -| 2.0.42 | fix wasNull interface return value in WebSocket connection | - | -| 2.0.41 | fix decode method of username and password in REST connection | - | -| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | - | -| 2.0.38 | JDBC REST connections add bulk pull function | - | -| 2.0.37 | Support json tags | - | -| 2.0.36 | Support schemaless writing | - | +| taos-jdbcdriver version | major changes | TDengine version | +| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | +| 3.2.1 | subscription add seek function | 3.0.5.0 or later | +| 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 or later | +| 3.2.0 | This version has been deprecated | - | +| 3.1.0 | JDBC REST connection supports subscription over WebSocket | - | +| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | - | +| 3.0.0 | Support for TDengine 3.0 | 3.0.0.0 or later | +| 2.0.42 | fix wasNull interface return value in WebSocket connection | - | +| 2.0.41 | fix decode method of username and password in REST connection | - | +| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | - | +| 2.0.38 | JDBC REST connections add bulk pull function | - | +| 2.0.37 | Support json tags | - | +| 2.0.36 | Support schemaless writing | - | **Note**: adding `batchfetch` to the REST connection and setting it to true will enable the WebSocket connection. diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index f63350ea04..1588159b57 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -34,20 +34,20 @@ REST 连接支持所有能运行 Java 的平台。 ## 版本历史 -| taos-jdbcdriver 版本 | 主要变化 | TDengine 最低版本 | -| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :---------------: | -| 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 | -| 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 | -| 3.2.0 | 存在连接问题，不推荐使用 | - | -| 3.1.0 | WebSocket 连接支持订阅功能 | - | -| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | - | -| 3.0.0 | 支持 TDengine 3.0 | 3.0.0.0 | -| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | - | -| 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | - | -| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | - | -| 2.0.38 | JDBC REST 连接增加批量拉取功能 | - | -| 2.0.37 | 增加对 json tag 支持 | - | -| 2.0.36 | 增加对 schemaless 写入支持 | - | +| taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | +| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | +| 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 及更高版本 | +| 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 及更高版本 | +| 3.2.0 | 存在连接问题，不推荐使用 | - | +| 3.1.0 | WebSocket 连接支持订阅功能 | - | +| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | - | +| 3.0.0 | 支持 TDengine 3.0 | 3.0.0.0 及更高版本 | +| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | - | +| 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | - | +| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | - | +| 2.0.38 | JDBC REST 连接增加批量拉取功能 | - | +| 2.0.37 | 增加对 json tag 支持 | - | +| 2.0.36 | 增加对 schemaless 写入支持 | - | **注**：REST 连接中增加 `batchfetch` 参数并设置为 true，将开启 WebSocket 连接。 diff --git a/examples/JDBC/springbootdemo/src/main/resources/application.properties b/examples/JDBC/springbootdemo/src/main/resources/application.properties index bf21047395..c523952fb6 100644 --- a/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -5,7 +5,7 @@ #spring.datasource.password=taosdata # datasource config - JDBC-RESTful spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -spring.datasource.url=jdbc:TAOS-RS://localhost:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.url=jdbc:TAOS-RS://localhost:6041/test spring.datasource.username=root spring.datasource.password=taosdata spring.datasource.druid.initial-size=5 From 029ca94ce5fcab837b0681e2d2f088e2785ae1a8 Mon Sep 17 00:00:00 2001 From: xleili Date: Thu, 8 Jun 2023 10:43:23 +0800 Subject: [PATCH 072/122] enhance: community package suport adapater release version --- tools/CMakeLists.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1461a7b373..dd8d55f0a8 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -121,20 +121,20 @@ ELSE () BUILD_COMMAND COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib - COMMAND go build -a -o taosadapter.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + # COMMAND go build -a -o taosadapter.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND -# COMMAND cmake -E echo "Comparessing taosadapter.exe" -# COMMAND cmake -E time upx taosadapter.exe + COMMAND cmake -E echo "Comparessing taosadapter.exe" + COMMAND cmake -E time upx taosadapter.exe COMMAND cmake -E echo "Copy taosadapter.exe" COMMAND cmake -E copy taosadapter.exe ${CMAKE_BINARY_DIR}/build/bin/taosadapter.exe COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ -# COMMAND cmake -E echo "Copy taosadapter-debug.exe" -# COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E echo "Copy taosadapter-debug.exe" + COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin ) ELSE (TD_WINDOWS) MESSAGE("Building taosAdapter on non-Windows") @@ -149,20 +149,20 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + # COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND -# COMMAND cmake -E echo "Comparessing taosadapter.exe" -# COMMAND upx taosadapter || : + COMMAND cmake -E echo "Comparessing taosadapter.exe" + COMMAND upx taosadapter || : COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ -# COMMAND cmake -E echo "Copy taosadapter-debug" -# COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E eKho "Copy taosadapter-debug" + COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) ENDIF (TD_WINDOWS) ENDIF () From 6c1f82c30e876eca2363570332f4968677087bcf Mon Sep 17 00:00:00 2001 From: xleili Date: Thu, 8 Jun 2023 17:27:35 +0800 Subject: [PATCH 073/122] fix: typo --- tools/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index dd8d55f0a8..9e8d91b6b6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -161,7 +161,7 @@ ELSE () COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ - COMMAND cmake -E eKho "Copy taosadapter-debug" + COMMAND cmake -E echo "Copy taosadapter-debug" COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) ENDIF (TD_WINDOWS) From 3f46a9bbc9e6e39f8eb363ee8ec667e51ad4dbe4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 8 Jun 2023 18:14:02 +0800 Subject: [PATCH 074/122] feat:add parameters for consumer & add offset rows for subscription --- include/common/tmsg.h | 56 +++-- include/util/tdef.h | 1 + source/client/src/clientTmq.c | 54 +++-- source/common/src/systable.c | 7 + source/common/src/tmsg.c | 47 ++++- source/dnode/mnode/impl/inc/mndDef.h | 17 +- source/dnode/mnode/impl/src/mndConsumer.c | 53 ++++- source/dnode/mnode/impl/src/mndDef.c | 234 +++++++++++++-------- source/dnode/mnode/impl/src/mndSubscribe.c | 48 +++-- source/dnode/vnode/src/tq/tq.c | 16 +- source/dnode/vnode/src/tq/tqUtil.c | 8 +- 11 files changed, 384 insertions(+), 157 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d78e771fcf..4b0a741213 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2033,6 +2033,12 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; char clientId[256]; SArray* topicNames; // SArray + + int8_t withTbName; + int8_t useSnapshot; + int8_t autoCommit; + int32_t autoCommitInterval; + int8_t resetOffsetCfg; } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -2047,6 +2053,13 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc for (int32_t i = 0; i < topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } + + tlen += taosEncodeFixedI8(buf, pReq->withTbName); + tlen += taosEncodeFixedI8(buf, pReq->useSnapshot); + tlen += taosEncodeFixedI8(buf, pReq->autoCommit); + tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval); + tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg); + return tlen; } @@ -2064,6 +2077,12 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); } + + buf = taosDecodeFixedI8(buf, &pReq->withTbName); + buf = taosDecodeFixedI8(buf, &pReq->useSnapshot); + buf = taosDecodeFixedI8(buf, &pReq->autoCommit); + buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval); + buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); return buf; } @@ -2455,15 +2474,6 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; } SMqAskEpReq; -typedef struct { - int64_t consumerId; - int32_t epoch; -} SMqHbReq; - -typedef struct { - int8_t reserved; -} SMqHbRsp; - typedef struct { int32_t key; int32_t valueLen; @@ -3354,6 +3364,28 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteMqSubTopicEp); } +typedef struct { + int32_t vgId; + STqOffsetVal offset; + int64_t rows; +}OffsetRows; + +typedef struct{ + char topicName[TSDB_TOPIC_FNAME_LEN]; + SArray* offsetRows; +}TopicOffsetRows; + +typedef struct { + int64_t consumerId; + int32_t epoch; + SArray* topics; +} SMqHbReq; + +typedef struct { + int8_t reserved; +} SMqHbRsp; + + #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { int64_t suid; @@ -3478,10 +3510,8 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); -int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); -int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); -int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); -int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeatroySMqHbReq(SMqHbReq* pReq); + #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 diff --git a/include/util/tdef.h b/include/util/tdef.h index 0b0569e2d1..2f566ef696 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -195,6 +195,7 @@ typedef enum ELogicConditionType { #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string #define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_OFFSET_LEN 80 // it is a null-terminated string #define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string #define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index e1b2b9c48b..2e3a634577 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -82,7 +82,7 @@ struct tmq_t { int8_t useSnapshot; int8_t autoCommit; int32_t autoCommitInterval; - int32_t resetOffsetCfg; + int8_t resetOffsetCfg; uint64_t consumerId; bool hbBgEnable; tmq_commit_cb* commitCb; @@ -567,10 +567,10 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN atomic_add_fetch_32(&pParamSet->totalRspNum, 1); SEp* pEp = GET_ACTIVE_EP(&pVg->epSet); - char offsetBuf[80] = {0}; + char offsetBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->offset.val); - char commitBuf[80] = {0}; + char commitBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, tmq->consumerId, pOffset->offset.subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, @@ -796,6 +796,21 @@ void tmqSendHbReq(void* param, void* tmrId) { SMqHbReq req = {0}; req.consumerId = tmq->consumerId; req.epoch = tmq->epoch; + req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); + for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){ + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + TopicOffsetRows* data = taosArrayReserve(req.topics, 1); + strcpy(data->topicName, pTopic->topicName); + data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); + for(int j = 0; j < numOfVgroups; j++){ + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + offRows->vgId = pVg->vgId; + offRows->rows = pVg->numOfRows; + offRows->offset = pVg->offsetInfo.committedOffset; + } + } int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); if (tlen < 0) { @@ -835,6 +850,7 @@ void tmqSendHbReq(void* param, void* tmrId) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); OVER: + tDeatroySMqHbReq(&req); taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer); taosReleaseRef(tmqMgmt.rsetId, refId); } @@ -1107,7 +1123,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } - char buf[80] = {0}; + char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; tFormatOffset(buf, tListLen(buf), &offset); tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 @@ -1123,7 +1139,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 60; // let's wait for 2 mins at most + const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1143,6 +1159,12 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } + req.withTbName = tmq->withTbName; + req.useSnapshot = tmq->useSnapshot; + req.autoCommit = tmq->autoCommit; + req.autoCommitInterval = tmq->autoCommitInterval; + req.resetOffsetCfg = tmq->resetOffsetCfg; + for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1375,8 +1397,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecoderClear(&decoder); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - char buf[80]; - tFormatOffset(buf, 80, &pRspWrapper->dataRsp.rspOffset); + char buf[TSDB_OFFSET_LEN]; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset); tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { @@ -1523,8 +1545,8 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); - char buf[80]; - tFormatOffset(buf, 80, &pVgCur->offsetInfo.currentOffset); + char buf[TSDB_OFFSET_LEN]; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); @@ -1745,7 +1767,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo->msgType = TDMT_VND_TMQ_CONSUME; int64_t transporterId = 0; - char offsetFormatBuf[80]; + char offsetFormatBuf[TSDB_OFFSET_LEN]; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, @@ -1882,8 +1904,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { pVg->offsetInfo.walVerEnd = pDataRsp->head.walever; pVg->receivedInfoFromVnode = true; - char buf[80]; - tFormatOffset(buf, 80, &pDataRsp->rspOffset); + char buf[TSDB_OFFSET_LEN]; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 " total:%" PRId64 " reqId:0x%" PRIx64, @@ -1990,8 +2012,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tmq->totalRows += numOfRows; - char buf[80]; - tFormatOffset(buf, 80, &pVg->offsetInfo.currentOffset); + char buf[TSDB_OFFSET_LEN]; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 " total:%" PRId64 " reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, @@ -2608,7 +2630,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; int64_t transporterId = 0; - char offsetFormatBuf[80]; + char offsetFormatBuf[TSDB_OFFSET_LEN]; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, @@ -2645,7 +2667,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG; - char offsetBuf[80] = {0}; + char offsetBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffsetInfo->currentOffset); tscDebug("vgId:%d offset is update to:%s", p->vgId, offsetBuf); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index db0cc78de6..21721d85a5 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -291,6 +291,8 @@ static const SSysDbTableSchema subscriptionSchema[] = { {.name = "consumer_group", .bytes = TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, + {.name = "offset", .bytes = TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, }; static const SSysDbTableSchema vnodesSchema[] = { @@ -359,6 +361,11 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "withTbName", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, + {.name = "useSnapshot", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, + {.name = "autoCommit", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, + {.name = "autoCommitInterval", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "resetOffsetCfg", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, }; static const SSysDbTableSchema offsetSchema[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index ac035e0a2b..f14375f9bf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5338,6 +5338,15 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { return 0; } +int32_t tDeatroySMqHbReq(SMqHbReq* pReq){ + for(int i = 0; i < taosArrayGetSize(pReq->topics); i++){ + TopicOffsetRows* vgs = taosArrayGet(pReq->topics, i); + if(vgs) taosArrayDestroy(vgs->offsetRows); + } + taosArrayDestroy(pReq->topics); + return 0; +} + int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -5346,6 +5355,21 @@ int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; if (tEncodeI32(&encoder, pReq->epoch) < 0) return -1; + int32_t sz = taosArrayGetSize(pReq->topics); + if (tEncodeI32(&encoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; ++i) { + TopicOffsetRows* vgs = (TopicOffsetRows*)taosArrayGet(pReq->topics, i); + if (tEncodeCStr(&encoder, vgs->topicName) < 0) return -1; + int32_t szVgs = taosArrayGetSize(vgs->offsetRows); + if (tEncodeI32(&encoder, szVgs) < 0) return -1; + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayGet(vgs->offsetRows, j); + if (tEncodeI32(&encoder, offRows->vgId) < 0) return -1; + if (tEncodeI64(&encoder, offRows->rows) < 0) return -1; + if (tEncodeSTqOffsetVal(&encoder, &offRows->offset) < 0) return -1; + } + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -5362,7 +5386,28 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; - + int32_t sz = 0; + if (tDecodeI32(&decoder, &sz) < 0) return -1; + if(sz > 0){ + pReq->topics = taosArrayInit(sz, sizeof(TopicOffsetRows)); + if (NULL == pReq->topics) return -1; + for (int32_t i = 0; i < sz; ++i) { + TopicOffsetRows* data = taosArrayReserve(pReq->topics, 1); + tDecodeCStrTo(&decoder, data->topicName); + int32_t szVgs = 0; + if (tDecodeI32(&decoder, &szVgs) < 0) return -1; + if(szVgs > 0){ + data->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); + if (NULL == data->offsetRows) return -1; + for (int32_t j= 0; j < szVgs; ++j) { + OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + if (tDecodeI32(&decoder, &offRows->vgId) < 0) return -1; + if (tDecodeI64(&decoder, &offRows->rows) < 0) return -1; + if (tDecodeSTqOffsetVal(&decoder, &offRows->offset) < 0) return -1; + } + } + } + } tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 82b714e6eb..ca0ee9e058 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -550,12 +550,18 @@ typedef struct { int64_t upTime; int64_t subscribeTime; int64_t rebalanceTime; + + int8_t withTbName; + int8_t useSnapshot; + int8_t autoCommit; + int32_t autoCommitInterval; + int32_t resetOffsetCfg; } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); -void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer); +void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); typedef struct { int32_t vgId; @@ -571,12 +577,13 @@ void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp); typedef struct { int64_t consumerId; // -1 for unassigned SArray* vgs; // SArray + SArray* offsetRows; // SArray } SMqConsumerEp; -SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp); -void tDeleteSMqConsumerEp(void* pEp); +//SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp); +//void tDeleteSMqConsumerEp(void* pEp); int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp); -void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp); +void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp, int8_t sver); typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -595,7 +602,7 @@ SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub); void tDeleteSubscribeObj(SMqSubscribeObj* pSub); int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); -void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub); +void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); typedef struct { int32_t epoch; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 117c1082a5..6e54e8c1b7 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -23,7 +23,7 @@ #include "tcompare.h" #include "tname.h" -#define MND_CONSUMER_VER_NUMBER 1 +#define MND_CONSUMER_VER_NUMBER 2 #define MND_CONSUMER_RESERVE_SIZE 64 #define MND_CONSUMER_LOST_HB_CNT 6 @@ -391,12 +391,13 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { + int32_t code = 0; SMnode *pMnode = pMsg->info.node; SMqHbReq req = {0}; - if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + if ((code = tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + goto end; } int64_t consumerId = req.consumerId; @@ -404,7 +405,8 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { if (pConsumer == NULL) { mError("consumer:0x%" PRIx64 " not exist", consumerId); terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; + code = -1; + goto end; } atomic_store_32(&pConsumer->hbStatus, 0); @@ -424,9 +426,25 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } + for(int i = 0; i < taosArrayGetSize(req.topics); i++){ + TopicOffsetRows* data = taosArrayGet(req.topics, i); + mDebug("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); + + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); + taosRLockLatch(&pSub->lock); + SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); + pConsumerEp->offsetRows = data->offsetRows; + data->offsetRows = NULL; + taosRUnLockLatch(&pSub->lock); + + mndReleaseSubscribe(pMnode, pSub); + } + mndReleaseConsumer(pMnode, pConsumer); - return 0; +end: + tDeatroySMqHbReq(&req); + return code; } static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { @@ -675,6 +693,12 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId)); + pConsumerNew->withTbName = subscribe.withTbName; + pConsumerNew->useSnapshot = subscribe.useSnapshot; + pConsumerNew->autoCommit = subscribe.autoCommit; + pConsumerNew->autoCommitInterval = subscribe.autoCommitInterval; + pConsumerNew->resetOffsetCfg = subscribe.resetOffsetCfg; + // set the update type pConsumerNew->updateType = CONSUMER_UPDATE__REBALANCE; taosArrayDestroy(pConsumerNew->assignedTopics); @@ -822,7 +846,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { goto CM_DECODE_OVER; } - if (sver != MND_CONSUMER_VER_NUMBER) { + if (sver < 1 || sver > MND_CONSUMER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto CM_DECODE_OVER; } @@ -849,7 +873,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); - if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) { + if (tDecodeSMqConsumerObj(buf, pConsumer, sver) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; // TODO set correct error code goto CM_DECODE_OVER; } @@ -1159,6 +1183,21 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->withTbName, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->useSnapshot, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->autoCommit, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->autoCommitInterval, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->resetOffsetCfg, false); + numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6dab018236..5c798b2fd8 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -321,10 +321,15 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { tlen += taosEncodeFixedI32(buf, 0); } + tlen += taosEncodeFixedI8(buf, pConsumer->withTbName); + tlen += taosEncodeFixedI8(buf, pConsumer->useSnapshot); + tlen += taosEncodeFixedI8(buf, pConsumer->autoCommit); + tlen += taosEncodeFixedI32(buf, pConsumer->autoCommitInterval); + tlen += taosEncodeFixedI32(buf, pConsumer->resetOffsetCfg); return tlen; } -void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { +void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t sver) { int32_t sz; buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); buf = taosDecodeStringTo(buf, pConsumer->clientId); @@ -375,50 +380,96 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { taosArrayPush(pConsumer->assignedTopics, &topic); } + if(sver > 1){ + buf = taosDecodeFixedI8(buf, &pConsumer->withTbName); + buf = taosDecodeFixedI8(buf, &pConsumer->useSnapshot); + buf = taosDecodeFixedI8(buf, &pConsumer->autoCommit); + buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval); + buf = taosDecodeFixedI32(buf, &pConsumer->resetOffsetCfg); + } + return (void *)buf; } -SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { - SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); - if (pConsumerEpNew == NULL) return NULL; - pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; - pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp); - return pConsumerEpNew; -} - -void tDeleteSMqConsumerEp(void *data) { - SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; - taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); -} +//SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { +// SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); +// if (pConsumerEpNew == NULL) return NULL; +// pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; +// pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp); +// return pConsumerEpNew; +//} +// +//void tDeleteSMqConsumerEp(void *data) { +// SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; +// taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); +//} int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); -#if 0 - int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - tlen += tEncodeSMqVgEp(buf, pVgEp); + int32_t szVgs = taosArrayGetSize(pConsumerEp->offsetRows); + tlen += taosEncodeFixedI32(buf, szVgs); + for (int32_t j= 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayGet(pConsumerEp->offsetRows, j); + tlen += taosEncodeFixedI32(buf, offRows->vgId); + tlen += taosEncodeFixedI64(buf, offRows->rows); + tlen += taosEncodeFixedI8(buf, offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + tlen += taosEncodeFixedI64(buf, offRows->offset.uid); + tlen += taosEncodeFixedI64(buf, offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + tlen += taosEncodeFixedI64(buf, offRows->offset.version); + } else { + // do nothing + } } -#endif +//#if 0 +// int32_t sz = taosArrayGetSize(pConsumerEp->vgs); +// tlen += taosEncodeFixedI32(buf, sz); +// for (int32_t i = 0; i < sz; i++) { +// SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); +// tlen += tEncodeSMqVgEp(buf, pVgEp); +// } +//#endif return tlen; } -void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) { +void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) { buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); -#if 0 - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *)); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); - buf = tDecodeSMqVgEp(buf, pVgEp); - taosArrayPush(pConsumerEp->vgs, &pVgEp); + if (sver > 1){ + int32_t szVgs = 0; + buf = taosDecodeFixedI32(buf, &szVgs); + if(szVgs > 0){ + pConsumerEp->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); + if (NULL == pConsumerEp->offsetRows) return NULL; + for (int32_t j= 0; j < szVgs; ++j) { + OffsetRows* offRows = taosArrayReserve(pConsumerEp->offsetRows, 1); + buf = taosDecodeFixedI32(buf, &offRows->vgId); + buf = taosDecodeFixedI64(buf, &offRows->rows); + buf = taosDecodeFixedI8(buf, &offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + buf = taosDecodeFixedI64(buf, &offRows->offset.uid); + buf = taosDecodeFixedI64(buf, &offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + buf = taosDecodeFixedI64(buf, &offRows->offset.version); + } else { + // do nothing + } + } + } } -#endif +//#if 0 +// int32_t sz; +// buf = taosDecodeFixedI32(buf, &sz); +// pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *)); +// for (int32_t i = 0; i < sz; i++) { +// SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); +// buf = tDecodeSMqVgEp(buf, pVgEp); +// taosArrayPush(pConsumerEp->vgs, &pVgEp); +// } +//#endif return (void *)buf; } @@ -479,6 +530,7 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); + taosArrayDestroy(pConsumerEp->offsetRows); } taosHashCleanup(pSub->consumerHash); taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp); @@ -511,7 +563,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { return tlen; } -void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { +void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { // buf = taosDecodeStringTo(buf, pSub->key); buf = taosDecodeFixedI64(buf, &pSub->dbUid); @@ -526,7 +578,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); for (int32_t i = 0; i < sz; i++) { SMqConsumerEp consumerEp = {0}; - buf = tDecodeSMqConsumerEp(buf, &consumerEp); + buf = tDecodeSMqConsumerEp(buf, &consumerEp, sver); taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp)); } @@ -535,65 +587,65 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { return (void *)buf; } -SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { - SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); - if (pEntryNew == NULL) return NULL; - pEntryNew->epoch = pEntry->epoch; - pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp); - return pEntryNew; -} +//SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { +// SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); +// if (pEntryNew == NULL) return NULL; +// pEntryNew->epoch = pEntry->epoch; +// pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp); +// return pEntryNew; +//} +// +//void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { +// taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp); +//} -void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { - taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp); -} +//int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { +// int32_t tlen = 0; +// tlen += taosEncodeFixedI32(buf, pEntry->epoch); +// tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); +// return tlen; +//} +// +//void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { +// buf = taosDecodeFixedI32(buf, &pEntry->epoch); +// buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); +// return (void *)buf; +//} -int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pEntry->epoch); - tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); - return tlen; -} +//SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { +// SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); +// if (pLogNew == NULL) return pLogNew; +// memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); +// pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp); +// return pLogNew; +//} +// +//void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { +// taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp); +//} -void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { - buf = taosDecodeFixedI32(buf, &pEntry->epoch); - buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); - return (void *)buf; -} - -SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { - SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); - if (pLogNew == NULL) return pLogNew; - memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); - pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp); - return pLogNew; -} - -void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { - taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp); -} - -int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pLog->key); - tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); - return tlen; -} - -void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { - buf = taosDecodeStringTo(buf, pLog->key); - buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); - return (void *)buf; -} - -int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pOffset->key); - tlen += taosEncodeFixedI64(buf, pOffset->offset); - return tlen; -} - -void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { - buf = taosDecodeStringTo(buf, pOffset->key); - buf = taosDecodeFixedI64(buf, &pOffset->offset); - return buf; -} +//int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { +// int32_t tlen = 0; +// tlen += taosEncodeString(buf, pLog->key); +// tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); +// return tlen; +//} +// +//void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { +// buf = taosDecodeStringTo(buf, pLog->key); +// buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); +// return (void *)buf; +//} +// +//int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { +// int32_t tlen = 0; +// tlen += taosEncodeString(buf, pOffset->key); +// tlen += taosEncodeFixedI64(buf, pOffset->offset); +// return tlen; +//} +// +//void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { +// buf = taosDecodeStringTo(buf, pOffset->key); +// buf = taosDecodeFixedI64(buf, &pOffset->offset); +// return buf; +//} diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 74421afa33..cc71f3c80d 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -24,7 +24,7 @@ #include "tcompare.h" #include "tname.h" -#define MND_SUBSCRIBE_VER_NUMBER 1 +#define MND_SUBSCRIBE_VER_NUMBER 2 #define MND_SUBSCRIBE_RESERVE_SIZE 64 #define MND_SUBSCRIBE_REBALANCE_CNT 3 @@ -809,7 +809,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; - if (sver != MND_SUBSCRIBE_VER_NUMBER) { + if (sver > MND_SUBSCRIBE_VER_NUMBER || sver < 1) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto SUB_DECODE_OVER; } @@ -828,7 +828,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); - if (tDecodeSubscribeObj(buf, pSub) == NULL) { + if (tDecodeSubscribeObj(buf, pSub, sver) == NULL) { goto SUB_DECODE_OVER; } @@ -1087,15 +1087,39 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId); // offset -#if 0 - // subscribe time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); - - // rebalance time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); -#endif + OffsetRows *data = NULL; + for(int i = 0; i < taosArrayGetSize(pConsumerEp->offsetRows); i++){ + OffsetRows *tmp = taosArrayGet(pConsumerEp->offsetRows, i); + if(data->vgId != pVgEp->vgId){ + continue; + } + data = tmp; + } + if(data){ + // vg id + char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); + varDataSetLen(buf, strlen(varDataVal(buf))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)buf, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&data->rows, false); + }else{ + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); + mError("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); + } +//#if 0 +// // subscribe time +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); +// +// // rebalance time +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); +//#endif numOfRows++; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aa6bbbe9df..0bb3b7575a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -243,8 +243,8 @@ int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId) { tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP, sver, ever); - char buf1[80] = {0}; - char buf2[80] = {0}; + char buf1[TSDB_OFFSET_LEN] = {0}; + char buf2[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset); tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset); tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", vgId, @@ -259,10 +259,10 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); - char buf1[80] = {0}; - char buf2[80] = {0}; - tFormatOffset(buf1, 80, &pRsp->reqOffset); - tFormatOffset(buf2, 80, &pRsp->rspOffset); + char buf1[TSDB_OFFSET_LEN] = {0}; + char buf2[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf1, TSDB_OFFSET_LEN, &pRsp->reqOffset); + tFormatOffset(buf2, TSDB_OFFSET_LEN, &pRsp->rspOffset); tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); @@ -481,8 +481,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { pHandle->epoch = reqEpoch; } - char buf[80]; - tFormatOffset(buf, 80, &reqOffset); + char buf[TSDB_OFFSET_LEN]; + tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index a34e765e50..e67b8906a9 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -99,8 +99,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand if (pOffset != NULL) { *pOffsetVal = pOffset->val; - char formatBuf[80]; - tFormatOffset(formatBuf, 80, pOffsetVal); + char formatBuf[TSDB_OFFSET_LEN]; + tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); @@ -186,8 +186,8 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); end : { - char buf[80] = {0}; - tFormatOffset(buf, 80, &dataRsp.rspOffset); + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); From 79625df5e4dac8f906e39ca3597c04df579f92b7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Jun 2023 18:39:51 +0800 Subject: [PATCH 075/122] chore: code optimization --- source/libs/planner/src/planLogicCreater.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 5a8512c12d..953ca13d3b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -732,25 +732,25 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return code; } -static int32_t eraseDuplicatedWindowPseudoCol(SWindowLogicNode* pWindow, SNodeList* pProjections) { +static int32_t eraseDuplicatedPseudoColumnFuncs(SNodeList* pFuncs, SNodeList* pProjections) { int32_t code = 0; int32_t funcIndex = 0; - SSHashObj* pHashFunc = NULL; + SSHashObj* pFuncHash = NULL; SNode* pFuncNode = NULL; - FOREACH(pFuncNode, pWindow->pFuncs) { + FOREACH(pFuncNode, pFuncs) { SFunctionNode* pFunc = (SFunctionNode*)pFuncNode; if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { ++funcIndex; continue; } - if (!pHashFunc && !(pHashFunc = tSimpleHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)))) { + if (!pFuncHash && !(pFuncHash = tSimpleHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)))) { code = TSDB_CODE_OUT_OF_MEMORY; break; } - void* hashVal = tSimpleHashGet(pHashFunc, &pFunc->funcId, sizeof(pFunc->funcId)); + void* hashVal = tSimpleHashGet(pFuncHash, &pFunc->funcId, sizeof(pFunc->funcId)); if (!hashVal) { - tSimpleHashPut(pHashFunc, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); + tSimpleHashPut(pFuncHash, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); ++funcIndex; continue; } @@ -765,13 +765,13 @@ static int32_t eraseDuplicatedWindowPseudoCol(SWindowLogicNode* pWindow, SNodeLi } } if (!exist) { - nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, funcIndex)); + nodesListErase(pFuncs, nodesListGetCell(pFuncs, funcIndex)); } else { - nodesListErase(pWindow->pFuncs, nodesListGetCell(pWindow->pFuncs, *(int32_t*)hashVal)); - tSimpleHashPut(pHashFunc, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); + nodesListErase(pFuncs, nodesListGetCell(pFuncs, *(int32_t*)hashVal)); + tSimpleHashPut(pFuncHash, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); } } - tSimpleHashCleanup(pHashFunc); + tSimpleHashCleanup(pFuncHash); return code; } @@ -794,7 +794,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm if (TSDB_CODE_SUCCESS == code && WINDOW_TYPE_INTERVAL == pWindow->winType && pSelect->pProjectionList) { // erase duplicated Window Pseudo funcNode by filtering colNode in pSelect->pProjectionList - code = eraseDuplicatedWindowPseudoCol(pWindow, pSelect->pProjectionList); + code = eraseDuplicatedPseudoColumnFuncs(pWindow->pFuncs, pSelect->pProjectionList); } if (TSDB_CODE_SUCCESS == code) { From ac5862c321b3f2c1fe54afe3910a4037c4c9057d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 8 Jun 2023 19:08:22 +0800 Subject: [PATCH 076/122] feat:add parameters for consumer & add offset rows for subscription --- source/dnode/mnode/impl/src/mndDef.c | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 5c798b2fd8..b7ad56d5d8 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -515,6 +515,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { SMqConsumerEp newEp = { .consumerId = pConsumerEp->consumerId, .vgs = taosArrayDup(pConsumerEp->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp), + .offsetRows = NULL, }; taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp)); } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index cc71f3c80d..acef882c20 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -255,7 +255,7 @@ static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pI for (int32_t i = 0; i < numOfNewConsumers; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); - SMqConsumerEp newConsumerEp; + SMqConsumerEp newConsumerEp = {0}; newConsumerEp.consumerId = consumerId; newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); From d7a4817124bf522cda05f0eb837561dd0db03414 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 8 Jun 2023 19:11:15 +0800 Subject: [PATCH 077/122] feat:add parameters for consumer & add offset rows for subscription --- source/dnode/mnode/impl/src/mndConsumer.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 6e54e8c1b7..4a6a37395d 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -433,8 +433,10 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); taosRLockLatch(&pSub->lock); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); - pConsumerEp->offsetRows = data->offsetRows; - data->offsetRows = NULL; + if(pConsumerEp){ + pConsumerEp->offsetRows = data->offsetRows; + data->offsetRows = NULL; + } taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); From d5c0ca3163160d568804e0b1b199829d507bce22 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 8 Jun 2023 13:36:52 +0000 Subject: [PATCH 078/122] opt batch write --- source/libs/stream/inc/streamBackendRocksdb.h | 5 ++ source/libs/stream/src/streamBackendRocksdb.c | 54 +++++++++++++------ source/libs/stream/src/tstreamFileState.c | 11 +++- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 1cbd7b042c..da4e442f1a 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -122,12 +122,17 @@ char* streamDefaultIterKey_rocksdb(void* iter, int32_t* len); char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len); // batch func +int streamStateGetCfIdx(SStreamState* pState, const char* funcName); void* streamStateCreateBatch(); int32_t streamStateGetBatchSize(void* pBatch); void streamStateClearBatch(void* pBatch); void streamStateDestroyBatch(void* pBatch); int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl); + +int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb_writebatch_t* pBatch, void* key, + void* val, int32_t vlen, int64_t ttl, void* tmpBuf); + int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index df045eef20..4ad208988b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -210,7 +210,6 @@ void streamBackendDelCompare(void* backend, void* arg) { } void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); -int streamGetInit(SStreamState* pState, const char* funcName); // |key|-----value------| // |key|ttl|len|userData| @@ -557,14 +556,20 @@ typedef struct { int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { SStreamValue key = {.unixTimestamp = ttl, .len = vlen, .data = (char*)(value)}; - - char* p = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int32_t) + key.len); - char* buf = p; - int32_t len = 0; - len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp); - len += taosEncodeFixedI32((void**)&buf, key.len); - len += taosEncodeBinary((void**)&buf, (char*)value, vlen); - *dest = p; + int32_t len = 0; + if (*dest == NULL) { + char* p = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int32_t) + key.len); + char* buf = p; + len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp); + len += taosEncodeFixedI32((void**)&buf, key.len); + len += taosEncodeBinary((void**)&buf, (char*)value, vlen); + *dest = p; + } else { + char* buf = *dest; + len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp); + len += taosEncodeFixedI32((void**)&buf, key.len); + len += taosEncodeBinary((void**)&buf, (char*)value, vlen); + } return len; } /* @@ -713,7 +718,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpts[i], tableOpt); params[i].tableOpt = tableOpt; - int idx = streamGetInit(NULL, funcname); + int idx = streamStateGetCfIdx(NULL, funcname); SCfInit* cfPara = &ginitDict[idx]; rocksdb_comparator_t* compare = @@ -744,7 +749,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t char idstr[128] = {0}; sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); - int idx = streamGetInit(NULL, funcname); + int idx = streamStateGetCfIdx(NULL, funcname); RocksdbCfInst* inst = NULL; RocksdbCfInst** pInst = taosHashGet(handle->cfInst, idstr, strlen(idstr) + 1); @@ -955,7 +960,7 @@ void streamStateDestroyCompar(void* arg) { taosMemoryFree(comp->comp); } -int streamGetInit(SStreamState* pState, const char* funcName) { +int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { int idx = -1; size_t len = strlen(funcName); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { @@ -1002,7 +1007,7 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt) { - int idx = streamGetInit(pState, cfName); + int idx = streamStateGetCfIdx(pState, cfName); if (snapshot != NULL) { *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(pState->pTdbState->rocksdb); @@ -1022,7 +1027,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(pState, funcname); \ + int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ @@ -1053,7 +1058,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(pState, funcname); \ + int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ @@ -1101,7 +1106,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(pState, funcname); \ + int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ code = -1; \ @@ -2041,7 +2046,7 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - int i = streamGetInit(pState, cfName); + int i = streamStateGetCfIdx(pState, cfName); if (i < 0) { qError("streamState failed to put to cf name:%s", cfName); @@ -2057,6 +2062,21 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr taosMemoryFree(ttlV); return 0; } +int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb_writebatch_t* pBatch, void* key, + void* val, int32_t vlen, int64_t ttl, void* tmpBuf) { + char buf[128] = {0}; + int32_t klen = ginitDict[cfIdx].enFunc((void*)key, buf); + char* ttlV = tmpBuf; + int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); + + rocksdb_column_family_handle_t* pCf = pState->pTdbState->pHandle[ginitDict[cfIdx].idx]; + rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); + + if (tmpBuf == NULL) { + taosMemoryFree(ttlV); + } + return 0; +} int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { char* err = NULL; rocksdb_write(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index bfaeca89f6..dc9a1f80bb 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -350,6 +350,11 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, const int32_t BATCH_LIMIT = 256; SListNode* pNode = NULL; + int idx = streamStateGetCfIdx(pFileState->pFileStore, "state"); + + int32_t len = pFileState->rowSize + sizeof(uint64_t) + sizeof(int32_t) + 1; + char* buf = taosMemoryCalloc(1, len); + void* batch = streamStateCreateBatch(); while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; @@ -360,9 +365,13 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, } SStateKey sKey = {.key = *((SWinKey*)pPos->pKey), .opNum = ((SStreamState*)pFileState->pFileStore)->number}; - code = streamStatePutBatch(pFileState->pFileStore, "state", batch, &sKey, pPos->pRowBuff, pFileState->rowSize, 0); + code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, &sKey, pPos->pRowBuff, pFileState->rowSize, + 0, buf); + memset(buf, 0, len); qDebug("===stream===put %" PRId64 " to disc, res %d", sKey.key.ts, code); } + taosMemoryFree(buf); + if (streamStateGetBatchSize(batch) > 0) { code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } From 5770a6f95f6229faf5748e3113d9373ac98c5389 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Fri, 9 Jun 2023 08:28:00 +0800 Subject: [PATCH 079/122] docs: upgrade connector-rust version --- docs/en/14-reference/03-connector/06-rust.mdx | 2 +- docs/zh/08-connector/26-rust.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index f32e32f2ad..344bd3590e 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -31,7 +31,7 @@ Websocket connections are supported on all platforms that can run Go. | connector-rust version | TDengine version | major features | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.8.8 | 3.0.5.0 or later | TMQ: Get consuming progress and seek offset to consume. | +| v0.8.10 | 3.0.5.0 or later | TMQ: Get consuming progress and seek offset to consume. | | v0.8.0 | 3.0.4.0 | Support schemaless insert. | | v0.7.6 | 3.0.3.0 | Support req_id in query. | | v0.6.0 | 3.0.0.0 | Base features. | diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index a02757b14e..c23228c8cf 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -30,7 +30,7 @@ Websocket 连接支持所有能运行 Rust 的平台。 | Rust 连接器版本 | TDengine 版本 | 主要功能 | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.8.8 | 3.0.5.0 or later | 消息订阅：获取消费进度及按照指定进度开始消费。 | +| v0.8.10 | 3.0.5.0 or later | 消息订阅：获取消费进度及按照指定进度开始消费。 | | v0.8.0 | 3.0.4.0 | 支持无模式写入。 | | v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 | | v0.6.0 | 3.0.0.0 | 基础功能。 | From d2b8ef1c740ec228e767e3e647d76f66b80b3f1a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 9 Jun 2023 09:20:38 +0800 Subject: [PATCH 080/122] fix: remove -y from taosdump test for 3.0 (#21663) * fix: remove -y from taosdump test * fix: taosdumpTestNanoSupport.py --- tests/pytest/tools/taosdumpTest.py | 6 +++--- tests/pytest/tools/taosdumpTest2.py | 6 +++--- tests/pytest/tools/taosdumpTestNanoSupport.py | 18 +++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index 4bfb7d5ba3..aafd365f34 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -92,9 +92,9 @@ class TDTestCase: else: tdLog.info("taosdump found: %s" % binPath) - os.system("%s -y --databases db -o ./taosdumptest/tmp1" % binPath) + os.system("%s --databases db -o ./taosdumptest/tmp1" % binPath) os.system( - "%s -y --databases db1 -o ./taosdumptest/tmp2" % + "%s --databases db1 -o ./taosdumptest/tmp2" % binPath) tdSql.execute("drop database db") @@ -172,7 +172,7 @@ class TDTestCase: tdSql.query("show stables") tdSql.checkRows(2) os.system( - "%s -y --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % + "%s --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % binPath) tdSql.execute("drop database db12312313231231321312312312_323") os.system("%s -i ./taosdumptest/tmp1" % binPath) diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py index 8a85ce10ed..36e0480e57 100644 --- a/tests/pytest/tools/taosdumpTest2.py +++ b/tests/pytest/tools/taosdumpTest2.py @@ -97,7 +97,7 @@ class TDTestCase: tdSql.query("show databases") tdSql.checkRows(2) - os.system("%s -i ./taosdumptest/tmp -y" % binPath) + os.system("%s -i ./taosdumptest/tmp" % binPath) tdSql.query("show databases") tdSql.checkRows(3) @@ -125,13 +125,13 @@ class TDTestCase: os.system("rm ./taosdumptest/tmp/*.sql") os.system("rm ./taosdumptest/tmp/*.avro*") os.system("rm -rf ./taosdumptest/tmp/taosdump.*") - os.system("%s -D test -o ./taosdumptest/tmp -y" % binPath) + os.system("%s -D test -o ./taosdumptest/tmp" % binPath) tdSql.execute("drop database test") tdSql.query("show databases") tdSql.checkRows(3) - os.system("%s -i ./taosdumptest/tmp -y" % binPath) + os.system("%s -i ./taosdumptest/tmp" % binPath) tdSql.execute("use test") tdSql.query("show stables") diff --git a/tests/pytest/tools/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdumpTestNanoSupport.py index c40462b8db..2a3990614a 100644 --- a/tests/pytest/tools/taosdumpTestNanoSupport.py +++ b/tests/pytest/tools/taosdumpTestNanoSupport.py @@ -134,15 +134,15 @@ class TDTestCase: # dump all data os.system( - "%s -y -g --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s -g --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) # dump part data with -S -E os.system( - '%s -y -g --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -o ./taosdumptest/dumptmp2 ' % + '%s -g --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % + '%s -g --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % binPath) tdSql.execute("drop database timedb1") @@ -200,14 +200,14 @@ class TDTestCase: self.createdb(precision="us") os.system( - "%s -y -g --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s -g --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000000 -E 1625068860000000 -o ./taosdumptest/dumptmp2 ' % + '%s -g --databases timedb1 -S 1625068810000000 -E 1625068860000000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % + '%s -g --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % binPath) os.system("%s -i ./taosdumptest/dumptmp1" % binPath) @@ -269,14 +269,14 @@ class TDTestCase: self.createdb(precision="ms") os.system( - "%s -y -g --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s -g --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000 -E 1625068860000 -o ./taosdumptest/dumptmp2 ' % + '%s -g --databases timedb1 -S 1625068810000 -E 1625068860000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % + '%s -g --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % binPath) os.system("%s -i ./taosdumptest/dumptmp1" % binPath) From a67ef8ce916fec2c41b7e573f5338305b9cda94a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 9 Jun 2023 01:27:08 +0000 Subject: [PATCH 081/122] opt batch write --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4ad208988b..c4e3f147e0 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -877,7 +877,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { taosThreadRwlockInit(&pState->pTdbState->rwLock, NULL); SCfComparator compare = {.comp = pCompare, .numOfComp = cfLen}; pState->pTdbState->pComparNode = streamBackendAddCompare(handle, &compare); - // rocksdb_writeoptions_disable_WAL(pState->pTdbState->writeOpts, 1); + rocksdb_writeoptions_disable_WAL(pState->pTdbState->writeOpts, 1); qInfo("succ to open state %p on backend, %p, 0x%" PRIx64 "-%d", pState, handle, pState->streamId, pState->taskId); return 0; } From 06b65ce5c1bbdf4632706cd84e2b24193b22b404 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 9 Jun 2023 07:49:43 +0000 Subject: [PATCH 082/122] opt batch write --- source/libs/stream/src/streamBackendRocksdb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c4e3f147e0..3581a4c0ff 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -89,6 +89,8 @@ void* streamBackendInit(const char* path) { pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); rocksdb_env_t* env = rocksdb_create_default_env(); // rocksdb_envoptions_create(); + rocksdb_env_set_low_priority_background_threads(env, tsNumOfSnodeStreamThreads); + rocksdb_env_set_high_priority_background_threads(env, tsNumOfSnodeStreamThreads); rocksdb_cache_t* cache = rocksdb_cache_create_lru(64 << 20); @@ -96,13 +98,13 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); - rocksdb_options_set_write_buffer_size(opts, 48 << 20); rocksdb_options_set_max_total_wal_size(opts, 128 << 20); rocksdb_options_set_recycle_log_file_num(opts, 6); - rocksdb_options_set_max_write_buffer_number(opts, 2); + rocksdb_options_set_max_write_buffer_number(opts, 3); rocksdb_options_set_info_log_level(opts, 0); uint32_t dbLimit = nextPow2(tsMaxStreamBackendCache); rocksdb_options_set_db_write_buffer_size(opts, dbLimit << 20); + rocksdb_options_set_write_buffer_size(opts, (dbLimit << 20) / 2); pHandle->env = env; pHandle->dbOpt = opts; From d38e3835c097b1697e120a6bdb6bd296c756be5f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 9 Jun 2023 08:12:39 +0000 Subject: [PATCH 083/122] opt batch write --- source/libs/stream/src/streamBackendRocksdb.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 3581a4c0ff..cebe4e8204 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -81,6 +81,8 @@ const char* compareParKeyName(void* name); const char* comparePartagKeyName(void* name); void* streamBackendInit(const char* path) { + uint32_t dbMemLimit = nextPow2(tsMaxStreamBackendCache) << 20; + qDebug("start to init stream backend at %s", path); SBackendHandle* pHandle = taosMemoryCalloc(1, sizeof(SBackendHandle)); pHandle->list = tdListNew(sizeof(SCfComparator)); @@ -89,22 +91,23 @@ void* streamBackendInit(const char* path) { pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); rocksdb_env_t* env = rocksdb_create_default_env(); // rocksdb_envoptions_create(); - rocksdb_env_set_low_priority_background_threads(env, tsNumOfSnodeStreamThreads); - rocksdb_env_set_high_priority_background_threads(env, tsNumOfSnodeStreamThreads); - rocksdb_cache_t* cache = rocksdb_cache_create_lru(64 << 20); + int32_t nBGThread = tsNumOfSnodeStreamThreads <= 2 ? 1 : tsNumOfSnodeStreamThreads / 2; + rocksdb_env_set_low_priority_background_threads(env, nBGThread); + rocksdb_env_set_high_priority_background_threads(env, nBGThread); + + rocksdb_cache_t* cache = rocksdb_cache_create_lru(dbMemLimit / 2); rocksdb_options_t* opts = rocksdb_options_create(); rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); - rocksdb_options_set_max_total_wal_size(opts, 128 << 20); + rocksdb_options_set_max_total_wal_size(opts, dbMemLimit); rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 3); rocksdb_options_set_info_log_level(opts, 0); - uint32_t dbLimit = nextPow2(tsMaxStreamBackendCache); - rocksdb_options_set_db_write_buffer_size(opts, dbLimit << 20); - rocksdb_options_set_write_buffer_size(opts, (dbLimit << 20) / 2); + rocksdb_options_set_db_write_buffer_size(opts, dbMemLimit); + rocksdb_options_set_write_buffer_size(opts, dbMemLimit / 2); pHandle->env = env; pHandle->dbOpt = opts; From 8e0dfe7371536b258a85fd6f5247fce0e1d77af2 Mon Sep 17 00:00:00 2001 From: huolibo Date: Fri, 9 Jun 2023 16:32:54 +0800 Subject: [PATCH 084/122] docs(kafka): output format description --- docs/en/20-third-party/11-kafka.md | 6 ++++-- docs/zh/20-third-party/11-kafka.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 1fc2b57a13..30f970096f 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -356,8 +356,10 @@ The following configuration items apply to TDengine Sink Connector and TDengine 3. `timestamp.initial`: Data synchronization start time. The format is 'yyyy-MM-dd HH:mm:ss'. If it is not set, the data importing to Kafka will be started from the first/oldest row in the database. 4. `poll.interval.ms`: The time interval for checking newly created tables or removed tables, default value is 1000. 5. `fetch.max.rows`: The maximum number of rows retrieved when retrieving the database, default is 100. -6. `query.interval.ms`: The time range of reading data from TDengine each time, its unit is millisecond. It should be adjusted according to the data flow in rate, the default value is 1000. -7. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is `--`; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is `-`. +6. `query.interval.ms`: The time range of reading data from TDengine each time, its unit is millisecond. It should be adjusted according to the data flow in rate, the default value is 0, this means to get all the data to the latest time. +7. `out.format`: Result output format. `line` indicates that the output format is InfluxDB line protocol format, `json` indicates that the output format is json. The default is line. +8. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is `--`; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is `-`. +9. `topic.ignore.db`: Whether the topic naming rule contains the database name: true indicates that the rule is `-`, false indicates that the rule is `--`, and the default is false. Does not take effect when `topic.per.stable` is set to false. ## Other notes diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 641e2d5174..02f2921a90 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -361,8 +361,10 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 3. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'，若未指定则从指定 DB 中最早的一条记录开始。 4. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 5. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 -6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 1000. -7. `topic.per.stable`: 如果设置为true，表示一个超级表对应一个 Kafka topic，topic的命名规则 `--`；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `-` +6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 +7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。 +8. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 `--`；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `-` +9. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 `-`，false 表示规则为 `--`，默认 false。在 `topic.per.stable` 设置为 false 时不生效。 ## 其他说明 From 1bf1714f53bea2eaf38c137b9f8f193c2aa6fb69 Mon Sep 17 00:00:00 2001 From: huolibo Date: Fri, 9 Jun 2023 16:40:26 +0800 Subject: [PATCH 085/122] docs: add out format default value --- docs/zh/20-third-party/11-kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 02f2921a90..679669198e 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -362,7 +362,7 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 4. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 5. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 -7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。 +7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 8. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 `--`；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `-` 9. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 `-`，false 表示规则为 `--`，默认 false。在 `topic.per.stable` 设置为 false 时不生效。 From 818e5ddf4144833b2b031341be08c5eb96a6b63e Mon Sep 17 00:00:00 2001 From: sunpeng Date: Fri, 9 Jun 2023 16:46:17 +0800 Subject: [PATCH 086/122] docs: upgrade docs from python subscription --- docs/en/07-develop/07-tmq.mdx | 6 + .../14-reference/03-connector/07-python.mdx | 132 ++++++++++++++++- docs/zh/07-develop/07-tmq.mdx | 6 + docs/zh/08-connector/30-python.mdx | 138 +++++++++++++++++- 4 files changed, 275 insertions(+), 7 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index d951923de5..578f38e73d 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -105,6 +105,12 @@ class Consumer: def poll(self, timeout: float = 1.0): pass + def assignment(self): + pass + + def poll(self, timeout: float = 1.0): + pass + def close(self): pass diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index ba322348cf..0bc888eedd 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -460,13 +460,77 @@ Connector support data subscription. For more information about subscroption, pl -The `consumer` in the connector has the subscription api. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/). +The `consumer` in the connector contains the subscription api. + +#### Create Consumer + +The syntax for creating a consumer is `consumer = Consumer(configs)`. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/). + +```python +from taos.tmq import Consumer + +consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) +``` + +#### Subscribe topics + +The `subscribe` function is used to subscribe to a list of topics. + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + +#### Consume + +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. + +```python +while True: + res = consumer.poll(1) + if not res: + continue + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) +``` + +#### assignment + +The `assignment` function is used to get the assignment of the topic. + +```python +assignments = consumer.assignment() +``` + +#### Seek + +The `seek` function is used to reset the assignment of the topic. + +```python +tp = TopicPartition(topic='topic1', partition=0, offset=0) +consumer.seek(tp) +``` + +#### After consuming data + +You should unsubscribe to the topics and close the consumer after consuming. + +```python +consumer.unsubscribe() +consumer.close() +``` + +#### Tmq subscription example ```python {{#include docs/examples/python/tmq_example.py}} ``` -There is an `assignment` API in the connector, which can get the assignment of the topic. And there is a `seek` api to reset the assignment of the topic. +#### assignment and seek example ```python {{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} @@ -478,11 +542,73 @@ There is an `assignment` API in the connector, which can get the assignment of t In addition to native connections, the connector also supports subscriptions via websockets. +#### Create Consumer + +The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). + +```python +import taosws + +consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +``` + +#### subscribe topics + +The `subscribe` function is used to subscribe to a list of topics. + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + +#### Consume + +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. + +```python +while True: + res = consumer.poll(timeout=1.0) + if not res: + continue + err = res.error() + if err is not None: + raise err + for block in message: + for row in block: + print(row) +``` + +#### assignment + +The `assignment` function is used to get the assignment of the topic. + +```python +assignments = consumer.assignment() +``` + +#### Seek + +The `seek` function is used to reset the assignment of the topic. + +```python +consumer.seek(topic='topic1', partition=0, offset=0) +``` + +#### After consuming data + +You should unsubscribe to the topics and close the consumer after consuming. + +```python +consumer.unsubscribe() +consumer.close() +``` + +#### Subscription example + ```python {{#include docs/examples/python/tmq_websocket_example.py}} ``` -There is an `assignment` API in the connector, which can get the assignment of the topic. And there is a `seek` api to reset the assignment of the topic. +#### Assignment and seek example ```python {{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index bfea926f53..a87a1f64f8 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -105,6 +105,12 @@ class Consumer: def poll(self, timeout: float = 1.0): pass + def assignment(self): + pass + + def seek(self, partition): + pass + def close(self): pass diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 5c310ff79f..10fb2238ee 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -456,18 +456,82 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 ### 数据订阅 -连接器支持数据订阅功能，数据订阅功能请参考 [数据订阅](../../develop/tmq/)。 +连接器支持数据订阅功能，数据订阅功能请参考 [数据订阅文档](../../develop/tmq/)。 -`Consumer` 提供了 Python 连接器订阅 TMQ 数据的 API，相关 API 定义请参考 [数据订阅文档](../../develop/tmq/#%E4%B8%BB%E8%A6%81%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84%E5%92%8C-api)。 +`Consumer` 提供了 Python 连接器订阅 TMQ 数据的 API。 + +#### 创建 Consumer + +创建 Consumer 语法为 `consumer = Consumer(configs)`，参数定义请参考 [数据订阅文档](../../develop/tmq/#%E5%88%9B%E5%BB%BA%E6%B6%88%E8%B4%B9%E8%80%85-consumer)。 + +```python +from taos.tmq import Consumer + +consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) +``` + +#### 订阅 topics + +Comsumer API 的 `subscribe` 方法用于订阅 topics，consumer 支持同时订阅多个 topic。 + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + +#### 消费数据 + +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 + +```python +while True: + res = consumer.poll(1) + if not res: + continue + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) +``` + +#### 获取消费进度 + +Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic 的消费进度，返回结果类型为 TopicPartition 列表。 + +```python +assignments = consumer.assignment() +``` + +#### 重置消费进度 + +Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位置，方法参数类型为 TopicPartition。 + +```python +tp = TopicPartition(topic='topic1', partition=0, offset=0) +consumer.seek(tp) +``` + +#### 结束消费 + +消费结束后，应当取消订阅，并关闭 Consumer。 + +```python +consumer.unsubscribe() +consumer.close() +``` + +#### tmq 订阅示例代码 ```python {{#include docs/examples/python/tmq_example.py}} ``` -连接器提供了 `assignment`接口，用于获取 topic assignment 的功能，可以查询订阅的 assignment 的消费进度，并提供 `seek` 接口，用于重置 topic 的 assignment。 +#### 获取和重置消费进度示例代码 ```python {{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} @@ -477,7 +541,71 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -除了原生的连接方式，Python 连接器还支持通过 websocket 订阅 TMQ 数据。 +除了原生的连接方式，Python 连接器还支持通过 websocket 订阅 TMQ 数据，使用 websocket 方式订阅 TMQ 数据需要安装 `taos-ws-py`。 + +taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。 + +#### 创建 Consumer + +创建 Consumer 语法为 `consumer = Consumer(conf=configs)`，使用时需要指定 `td.connect.websocket.scheme` 参数值为 "ws"，参数定义请参考 [数据订阅文档](../../develop/tmq/#%E5%88%9B%E5%BB%BA%E6%B6%88%E8%B4%B9%E8%80%85-consumer)。 + +```python +import taosws + +consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +``` + +#### 订阅 topics + +Comsumer API 的 `subscribe` 方法用于订阅 topics，consumer 支持同时订阅多个 topic。 + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + +#### 消费数据 + +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 + +```python +while True: + res = consumer.poll(timeout=1.0) + if not res: + continue + err = res.error() + if err is not None: + raise err + for block in message: + for row in block: + print(row) +``` + +#### 获取消费进度 + +Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic 的消费进度，返回结果类型为 TopicPartition 列表。 + +```python +assignments = consumer.assignment() +``` + +#### 重置消费进度 + +Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位置。 + +```python +consumer.seek(topic='topic1', partition=0, offset=0) +``` + +#### 结束消费 + +消费结束后，应当取消订阅，并关闭 Consumer。 + +```python +consumer.unsubscribe() +consumer.close() +``` + +#### tmq 订阅示例代码 ```python {{#include docs/examples/python/tmq_websocket_example.py}} @@ -485,6 +613,8 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 连接器提供了 `assignment` 接口，用于获取 topic assignment 的功能，可以查询订阅的 topic 的消费进度，并提供 `seek` 接口，用于重置 topic 的消费进度。 +#### 获取和重置消费进度示例代码 + ```python {{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} ``` From b870d48cce9d4c6fc31a6a81d6a658e5e950d63a Mon Sep 17 00:00:00 2001 From: huolibo Date: Fri, 9 Jun 2023 17:20:58 +0800 Subject: [PATCH 087/122] fix: add paramters for example --- docs/en/20-third-party/11-kafka.md | 2 ++ docs/zh/20-third-party/11-kafka.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 30f970096f..2302bb5223 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -241,6 +241,8 @@ Input following content: "poll.interval.ms": 1000, "fetch.max.rows": 100, "topic.per.stable": true, + "topic.ignore.db": false, + "out.format": "line", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "value.converter": "org.apache.kafka.connect.storage.StringConverter" } diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 679669198e..e868ebcf34 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -240,6 +240,8 @@ vi source-demo.json "poll.interval.ms": 1000, "fetch.max.rows": 100, "topic.per.stable": true, + "topic.ignore.db": false, + "out.format": "line", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "value.converter": "org.apache.kafka.connect.storage.StringConverter" } From 9034a9a37584724ec1e229661cb6e3e5d02f87af Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 9 Jun 2023 17:22:27 +0800 Subject: [PATCH 088/122] docs: fix broken links --- docs/en/14-reference/03-connector/07-python.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index 0bc888eedd..6bd02644d4 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -455,7 +455,7 @@ As the way to connect introduced above but add `req_id` argument. ### Subscription -Connector support data subscription. For more information about subscroption, please refer to [Data Subscription](../../develop/tmq/). +Connector support data subscription. For more information about subscroption, please refer to [Data Subscription](../../../develop/tmq/). @@ -464,7 +464,7 @@ The `consumer` in the connector contains the subscription api. #### Create Consumer -The syntax for creating a consumer is `consumer = Consumer(configs)`. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/). +The syntax for creating a consumer is `consumer = Consumer(configs)`. For more subscription api parameters, please refer to [Data Subscription](../../../develop/tmq/). ```python from taos.tmq import Consumer @@ -544,7 +544,7 @@ In addition to native connections, the connector also supports subscriptions via #### Create Consumer -The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). +The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../../develop/tmq/#create-a-consumer). ```python import taosws From aa610b27a48432426e49dec259eafa896c46780b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 9 Jun 2023 18:51:10 +0800 Subject: [PATCH 089/122] feat:add parameters for consumer & add offset rows for subscription --- source/client/src/clientTmq.c | 12 +- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndDef.c | 42 ++- source/dnode/mnode/impl/src/mndSubscribe.c | 202 +++++------ .../system-test/7-tmq/checkOffsetRowParams.py | 313 ++++++++++++++++++ tests/system-test/7-tmq/tmqCommon.py | 2 +- 6 files changed, 454 insertions(+), 118 deletions(-) create mode 100644 tests/system-test/7-tmq/checkOffsetRowParams.py diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 2e3a634577..c15f6d55bc 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -809,6 +809,7 @@ void tmqSendHbReq(void* param, void* tmrId) { offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; offRows->offset = pVg->offsetInfo.committedOffset; + tscDebug("report row:%lldd, offset:%" PRId64, offRows->rows, offRows->offset.version); } } @@ -1695,7 +1696,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, return pRspObj; } -SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) { +SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); pRspObj->resType = RES_TYPE__TMQ_METADATA; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1710,6 +1711,13 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } + // extract the rows in this data packet + for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); + int64_t rows = htobe64(pRetrieve->numOfRows); + pVg->numOfRows += rows; + (*numOfRows) += rows; + } return pRspObj; } @@ -2007,7 +2015,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->taosxRsp.createTableNum == 0) { pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); } else { - pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper); + pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); } tmq->totalRows += numOfRows; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ca0ee9e058..f74a593b00 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -595,6 +595,7 @@ typedef struct { int64_t stbUid; SHashObj* consumerHash; // consumerId -> SMqConsumerEp SArray* unassignedVgs; // SArray + SArray* offsetRows; char dbName[TSDB_DB_FNAME_LEN]; } SMqSubscribeObj; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b7ad56d5d8..b6fb96882d 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -515,7 +515,6 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { SMqConsumerEp newEp = { .consumerId = pConsumerEp->consumerId, .vgs = taosArrayDup(pConsumerEp->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp), - .offsetRows = NULL, }; taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp)); } @@ -535,6 +534,7 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { } taosHashCleanup(pSub->consumerHash); taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp); + taosArrayDestroy(pSub->offsetRows); } int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { @@ -561,6 +561,23 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { if (cnt != sz) return -1; tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp); tlen += taosEncodeString(buf, pSub->dbName); + + int32_t szVgs = taosArrayGetSize(pSub->offsetRows); + tlen += taosEncodeFixedI32(buf, szVgs); + for (int32_t j= 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayGet(pSub->offsetRows, j); + tlen += taosEncodeFixedI32(buf, offRows->vgId); + tlen += taosEncodeFixedI64(buf, offRows->rows); + tlen += taosEncodeFixedI8(buf, offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + tlen += taosEncodeFixedI64(buf, offRows->offset.uid); + tlen += taosEncodeFixedI64(buf, offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + tlen += taosEncodeFixedI64(buf, offRows->offset.version); + } else { + // do nothing + } + } return tlen; } @@ -585,6 +602,29 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); buf = taosDecodeStringTo(buf, pSub->dbName); + + if (sver > 1){ + int32_t szVgs = 0; + buf = taosDecodeFixedI32(buf, &szVgs); + if(szVgs > 0){ + pSub->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); + if (NULL == pSub->offsetRows) return NULL; + for (int32_t j= 0; j < szVgs; ++j) { + OffsetRows* offRows = taosArrayReserve(pSub->offsetRows, 1); + buf = taosDecodeFixedI32(buf, &offRows->vgId); + buf = taosDecodeFixedI64(buf, &offRows->rows); + buf = taosDecodeFixedI8(buf, &offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + buf = taosDecodeFixedI64(buf, &offRows->offset.uid); + buf = taosDecodeFixedI64(buf, &offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + buf = taosDecodeFixedI64(buf, &offRows->offset.version); + } else { + // do nothing + } + } + } + } return (void *)buf; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index acef882c20..cadd1f53f0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -449,8 +449,26 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pRemovedIter; taosArrayPush(pOutput->rebVgs, pRebOutput); - if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed, put all vg into unassigned - taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); + if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed + taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows + if(pSub){ + taosRLockLatch(&pSub->lock); + if(pOutput->pSub->offsetRows == NULL){ + pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); + }else{ + taosArrayClear(pOutput->pSub->offsetRows); + } + pIter = NULL; + while(1){ + pIter = taosHashIterate(pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + taosArrayAddAll(pOutput->pSub->offsetRows, pConsumerEp->offsetRows); + } + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } } } @@ -890,6 +908,10 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc pOldSub->unassignedVgs = pNewSub->unassignedVgs; pNewSub->unassignedVgs = tmp1; + SArray *tmp2 = pOldSub->offsetRows; + pOldSub->offsetRows = pNewSub->offsetRows; + pNewSub->offsetRows = tmp2; + taosWUnLockLatch(&pOldSub->lock); return 0; } @@ -1028,6 +1050,61 @@ END: return code; } +static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t consumerId, const char* topic, const char* cgroup, SArray* vgs, SArray *offsetRows){ + int32_t sz = taosArrayGetSize(vgs); + for (int32_t j = 0; j < sz; j++) { + SMqVgEp *pVgEp = taosArrayGetP(vgs, j); + + SColumnInfoData *pColInfo; + int32_t cols = 0; + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)topic, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)cgroup, false); + + // vg id + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false); + + // consumer id + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)&consumerId, consumerId == -1); + + mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), + consumerId, varDataVal(cgroup), pVgEp->vgId); + + // offset + OffsetRows *data = NULL; + for(int i = 0; i < taosArrayGetSize(offsetRows); i++){ + OffsetRows *tmp = taosArrayGet(offsetRows, i); + if(tmp->vgId != pVgEp->vgId){ + continue; + } + data = tmp; + } + if(data){ + // vg id + char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); + varDataSetLen(buf, strlen(varDataVal(buf))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false); + }else{ + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, *numOfRows); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, *numOfRows); + mError("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); + } + (*numOfRows)++; + } + return 0; +} + int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -1048,6 +1125,13 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock blockDataEnsureCapacity(pBlock, numOfRows + pSub->vgNum); } + // topic and cgroup + char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false); + varDataSetLen(topic, strlen(varDataVal(topic))); + varDataSetLen(cgroup, strlen(varDataVal(cgroup))); + SMqConsumerEp *pConsumerEp = NULL; void *pIter = NULL; while (1) { @@ -1055,121 +1139,11 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock if (pIter == NULL) break; pConsumerEp = (SMqConsumerEp *)pIter; - int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - for (int32_t j = 0; j < sz; j++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - - SColumnInfoData *pColInfo; - int32_t cols = 0; - - // topic and cgroup - char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false); - varDataSetLen(topic, strlen(varDataVal(topic))); - varDataSetLen(cgroup, strlen(varDataVal(cgroup))); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); - - // vg id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); - - // consumer id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false); - - mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), - pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId); - - // offset - OffsetRows *data = NULL; - for(int i = 0; i < taosArrayGetSize(pConsumerEp->offsetRows); i++){ - OffsetRows *tmp = taosArrayGet(pConsumerEp->offsetRows, i); - if(data->vgId != pVgEp->vgId){ - continue; - } - data = tmp; - } - if(data){ - // vg id - char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; - tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); - varDataSetLen(buf, strlen(varDataVal(buf))); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)buf, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&data->rows, false); - }else{ - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetNULL(pColInfo, numOfRows); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetNULL(pColInfo, numOfRows); - mError("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); - } -//#if 0 -// // subscribe time -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); -// -// // rebalance time -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); -//#endif - - numOfRows++; - } + buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, pConsumerEp->offsetRows); } // do not show for cleared subscription - int32_t sz = taosArrayGetSize(pSub->unassignedVgs); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); - - SColumnInfoData *pColInfo; - int32_t cols = 0; - - // topic and cgroup - char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false); - varDataSetLen(topic, strlen(varDataVal(topic))); - varDataSetLen(cgroup, strlen(varDataVal(cgroup))); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); - - // vg id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); - - // consumer id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, NULL, true); - - mDebug("mnd show subscriptions(unassigned): topic %s, cgroup %s vgid %d", varDataVal(topic), varDataVal(cgroup), - pVgEp->vgId); - - // offset -#if 0 - // subscribe time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); - - // rebalance time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); -#endif - - numOfRows++; - } + buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows); pBlock->info.rows = numOfRows; diff --git a/tests/system-test/7-tmq/checkOffsetRowParams.py b/tests/system-test/7-tmq/checkOffsetRowParams.py new file mode 100644 index 0000000000..8e5ff63f60 --- /dev/null +++ b/tests/system-test/7-tmq/checkOffsetRowParams.py @@ -0,0 +1,313 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"], \ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase1(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 1: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db1', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:2000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 10 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + time.sleep(2) + tdLog.info("start show subscriptions 1") + while(1): + tdSql.query("show subscriptions") + if (tdSql.getRows() == 0): + tdLog.info("sleep") + time.sleep(1) + elif (tdSql.queryResult[0][4] != None): + tdSql.checkData(0, 4, "offset(reset to earlieast)") + tdSql.checkData(0, 5, 0) + break + + tdLog.info("start insert data") + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + time.sleep(2) + tdLog.info("start show subscriptions 2") + tdSql.query("show subscriptions") + tdSql.checkRows(4) + print(tdSql.queryResult) + # tdSql.checkData(0, 4, 'offset(log) ver:103') + tdSql.checkData(0, 5, 10000) + # tdSql.checkData(1, 4, 'offset(log) ver:103') + tdSql.checkData(1, 5, 10000) + # tdSql.checkData(2, 4, 'offset(log) ver:303') + tdSql.checkData(2, 5, 50000) + # tdSql.checkData(3, 4, 'offset(log) ver:239') + tdSql.checkData(3, 5, 30000) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + + time.sleep(2) + tdLog.info("start show subscriptions 3") + tdSql.query("show subscriptions") + tdSql.checkRows(4) + print(tdSql.queryResult) + tdSql.checkData(0, 3, None) + # tdSql.checkData(0, 4, 'offset(log) ver:103') + tdSql.checkData(0, 5, 10000) + # tdSql.checkData(1, 4, 'offset(log) ver:103') + tdSql.checkData(1, 5, 10000) + # tdSql.checkData(2, 4, 'offset(log) ver:303') + tdSql.checkData(2, 5, 50000) + # tdSql.checkData(3, 4, 'offset(log) ver:239') + tdSql.checkData(3, 5, 30000) + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase1(cfgPath, buildPath) + # self.tmqCase2(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 6b633fa193..97f38f01b6 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -171,7 +171,7 @@ class TMQCom: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return From c492dd86f6a7cf408d2f9443257bbe821a0a638f Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 12 Jun 2023 09:05:51 +0800 Subject: [PATCH 090/122] docs(kafka): add topic delimiter parameter --- docs/en/20-third-party/11-kafka.md | 14 ++++++++------ docs/zh/20-third-party/11-kafka.md | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 2302bb5223..f643583312 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -215,7 +215,7 @@ The role of the TDengine Source Connector is to push all the data of a specific TDengine Source Connector will convert the data in TDengine data table into [InfluxDB Line protocol format](/develop/insert-data/influxdb-line/) or [OpenTSDB JSON protocol format](/develop/insert-data/opentsdb-json ) and then write to Kafka. -The following sample program synchronizes the data in the database test to the topic tdengine-source-test. +The following sample program synchronizes the data in the database test to the topic tdengine-test-meters. ### Add Source Connector configuration file @@ -237,7 +237,8 @@ Input following content: "connection.database": "test", "connection.attempts": 3, "connection.backoff.ms": 5000, - "topic.prefix": "tdengine-source", + "topic.prefix": "tdengine", + "topic.delimiter": "-", "poll.interval.ms": 1000, "fetch.max.rows": 100, "topic.per.stable": true, @@ -283,10 +284,10 @@ curl -X POST -d @source-demo.json http://localhost:8083/connectors -H "Content-T ### View topic data -Use the kafka-console-consumer command-line tool to monitor data in the topic tdengine-source-test. In the beginning, all historical data will be output. After inserting two new data into TDengine, kafka-console-consumer immediately outputs the two new data. The output is in InfluxDB line protocol format. +Use the kafka-console-consumer command-line tool to monitor data in the topic tdengine-test-meters. In the beginning, all historical data will be output. After inserting two new data into TDengine, kafka-console-consumer immediately outputs the two new data. The output is in InfluxDB line protocol format. ````shell -kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test-meters +kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-test-meters ```` output: @@ -360,8 +361,9 @@ The following configuration items apply to TDengine Sink Connector and TDengine 5. `fetch.max.rows`: The maximum number of rows retrieved when retrieving the database, default is 100. 6. `query.interval.ms`: The time range of reading data from TDengine each time, its unit is millisecond. It should be adjusted according to the data flow in rate, the default value is 0, this means to get all the data to the latest time. 7. `out.format`: Result output format. `line` indicates that the output format is InfluxDB line protocol format, `json` indicates that the output format is json. The default is line. -8. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is `--`; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is `-`. -9. `topic.ignore.db`: Whether the topic naming rule contains the database name: true indicates that the rule is `-`, false indicates that the rule is `--`, and the default is false. Does not take effect when `topic.per.stable` is set to false. +8. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is ``; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is ``. +9. `topic.ignore.db`: Whether the topic naming rule contains the database name: true indicates that the rule is ``, false indicates that the rule is ``, and the default is false. Does not take effect when `topic.per.stable` is set to false. +10. `topic.delimiter`: topic name delimiter，default is `-`。 ## Other notes diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index e868ebcf34..76e546c345 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -214,7 +214,7 @@ TDengine Source Connector 的作用是将 TDengine 某个数据库某一时刻 TDengine Source Connector 会将 TDengine 数据表中的数据转换成 [InfluxDB Line 协议格式](/develop/insert-data/influxdb-line/) 或 [OpenTSDB JSON 协议格式](/develop/insert-data/opentsdb-json)， 然后写入 Kafka。 -下面的示例程序同步数据库 test 中的数据到主题 tdengine-source-test。 +下面的示例程序同步数据库 test 中的数据到主题 tdengine-test-meters。 ### 添加 Source Connector 配置文件 @@ -236,7 +236,8 @@ vi source-demo.json "connection.database": "test", "connection.attempts": 3, "connection.backoff.ms": 5000, - "topic.prefix": "tdengine-source", + "topic.prefix": "tdengine", + "topic.delimiter": "-", "poll.interval.ms": 1000, "fetch.max.rows": 100, "topic.per.stable": true, @@ -282,10 +283,10 @@ curl -X POST -d @source-demo.json http://localhost:8083/connectors -H "Content-T ### 查看 topic 数据 -使用 kafka-console-consumer 命令行工具监控主题 tdengine-source-test 中的数据。一开始会输出所有历史数据， 往 TDengine 插入两条新的数据之后，kafka-console-consumer 也立即输出了新增的两条数据。 输出数据 InfluxDB line protocol 的格式。 +使用 kafka-console-consumer 命令行工具监控主题 tdengine-test-meters 中的数据。一开始会输出所有历史数据， 往 TDengine 插入两条新的数据之后，kafka-console-consumer 也立即输出了新增的两条数据。 输出数据 InfluxDB line protocol 的格式。 ```shell -kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test-meters +kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-test-meters ``` 输出： @@ -365,8 +366,9 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 5. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 -8. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 `--`；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `-` -9. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 `-`，false 表示规则为 `--`，默认 false。在 `topic.per.stable` 设置为 false 时不生效。 +8. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` +9. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 +10. `topic.delimiter`: topic 名称分割符，默认为 `-`。 ## 其他说明 From 58c43901ede6b45a2f248d43c373e419a6c56d4b Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jun 2023 09:14:38 +0800 Subject: [PATCH 091/122] chore: another logic --- source/libs/parser/src/parTranslater.c | 6 +++ source/libs/planner/src/planLogicCreater.c | 48 ---------------------- 2 files changed, 6 insertions(+), 48 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3b94bae05c..5f3e402027 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3108,6 +3108,12 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { if ('\0' == pExpr->userAlias[0]) { strcpy(pExpr->userAlias, pExpr->aliasName); } + if (QUERY_NODE_COLUMN == nodeType(pProject) && + ((0 == strcasecmp("_wstart", pExpr->userAlias) || 0 == strcasecmp("_wend", pExpr->userAlias) || + 0 == strcasecmp("_wduration", pExpr->userAlias)) && + '\0' != pExpr->aliasName[0])) { + continue; + } sprintf(pExpr->aliasName, "#expr_%d", no++); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 953ca13d3b..5bbc9acdad 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -732,49 +732,6 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return code; } -static int32_t eraseDuplicatedPseudoColumnFuncs(SNodeList* pFuncs, SNodeList* pProjections) { - int32_t code = 0; - int32_t funcIndex = 0; - SSHashObj* pFuncHash = NULL; - SNode* pFuncNode = NULL; - FOREACH(pFuncNode, pFuncs) { - SFunctionNode* pFunc = (SFunctionNode*)pFuncNode; - if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { - ++funcIndex; - continue; - } - if (!pFuncHash && !(pFuncHash = tSimpleHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)))) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; - } - - void* hashVal = tSimpleHashGet(pFuncHash, &pFunc->funcId, sizeof(pFunc->funcId)); - if (!hashVal) { - tSimpleHashPut(pFuncHash, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); - ++funcIndex; - continue; - } - - bool exist = false; - SNode* pProject = NULL; - FOREACH(pProject, pProjections) { - if (QUERY_NODE_COLUMN == nodeType(pProject) && 0 != pFunc->node.aliasName[0] && - 0 == strncmp(pFunc->node.aliasName, ((SColumnNode*)pProject)->colName, TSDB_COL_NAME_LEN)) { - exist = true; - break; - } - } - if (!exist) { - nodesListErase(pFuncs, nodesListGetCell(pFuncs, funcIndex)); - } else { - nodesListErase(pFuncs, nodesListGetCell(pFuncs, *(int32_t*)hashVal)); - tSimpleHashPut(pFuncHash, &pFunc->funcId, sizeof(pFunc->funcId), &funcIndex, sizeof(funcIndex)); - } - } - tSimpleHashCleanup(pFuncHash); - return code; -} - static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { if (pCxt->pPlanCxt->streamQuery) { @@ -792,11 +749,6 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL); } - if (TSDB_CODE_SUCCESS == code && WINDOW_TYPE_INTERVAL == pWindow->winType && pSelect->pProjectionList) { - // erase duplicated Window Pseudo funcNode by filtering colNode in pSelect->pProjectionList - code = eraseDuplicatedPseudoColumnFuncs(pWindow->pFuncs, pSelect->pProjectionList); - } - if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } From e50339045f8115518f058dd2a5d7271df84698e1 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 12 Jun 2023 09:12:14 +0800 Subject: [PATCH 092/122] doc: refine the location of taos.cfg --- docs/en/14-reference/12-config/index.md | 2 +- docs/zh/14-reference/12-config/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 52ded6208a..2d290a5f49 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -5,7 +5,7 @@ description: This document describes the configuration parameters for the TDengi ## Configuration File on Server Side -On the server side, the actual service of TDengine is provided by an executable `taosd` whose parameters can be configured in file `taos.cfg` to meet the requirements of different use cases. The default location of `taos.cfg` is `/etc/taos`, but can be changed by using `-c` parameter on the CLI of `taosd`. For example, the configuration file can be put under `/home/user` and used like below +On the server side, the actual service of TDengine is provided by an executable `taosd` whose parameters can be configured in file `taos.cfg` to meet the requirements of different use cases. The default location of `taos.cfg` is `/etc/taos` on Linux system, it's located under `C:\TDengine` on Windows system. The location of configuration file can be specified by using `-c` parameter on the CLI of `taosd`. For example, on Linux system the configuration file can be put under `/home/user` and used like below ``` taosd -c /home/user diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 68f44d1e65..115d0ca2c7 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -5,7 +5,7 @@ description: "TDengine 客户端和服务配置列表" ## 为服务端指定配置文件 -TDengine 系统后台服务由 taosd 提供，可以在配置文件 taos.cfg 里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos 目录，可以通过 taosd 命令行执行参数 -c 指定配置文件目录。比如，指定配置文件位于`/home/user` 这个目录： +TDengine 系统后台服务由 taosd 提供，可以在配置文件 taos.cfg 里修改配置参数，以满足不同场景的需求。在 Linux 系统上，配置文件的缺省位置在 `/etc/taos` 目录，在 Windows 系统上缺省位置在 `C:\TDengine` 。可以通过 taosd 命令行执行参数 -c 指定配置文件所在目录。比如，在 Linux 系统上可以指定配置文件位于 `/home/user` 这个目录： ``` taosd -c /home/user From 6657b8d8499ed18f16e1bcddc45d961a9cfc8088 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 09:44:00 +0800 Subject: [PATCH 093/122] fix:modify offset description to string --- include/common/tmsg.h | 2 +- include/util/tdef.h | 2 +- source/client/src/clientTmq.c | 5 ++--- source/common/src/systable.c | 10 +++++----- source/common/src/tmsg.c | 14 +++++++------- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++++++- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqUtil.c | 2 +- 8 files changed, 25 insertions(+), 21 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4b0a741213..aa0a243e68 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2901,7 +2901,7 @@ int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pRe // tqOffset enum { TMQ_OFFSET__RESET_NONE = -3, - TMQ_OFFSET__RESET_EARLIEAST = -2, + TMQ_OFFSET__RESET_EARLIEST = -2, TMQ_OFFSET__RESET_LATEST = -1, TMQ_OFFSET__LOG = 1, TMQ_OFFSET__SNAPSHOT_DATA = 2, diff --git a/include/util/tdef.h b/include/util/tdef.h index 2f566ef696..8122f94a11 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -195,7 +195,7 @@ typedef enum ELogicConditionType { #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string #define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_OFFSET_LEN 80 // it is a null-terminated string +#define TSDB_OFFSET_LEN 64 // it is a null-terminated string #define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string #define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c15f6d55bc..cb38996ea1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -264,7 +264,7 @@ tmq_conf_t* tmq_conf_new() { conf->withTbName = false; conf->autoCommit = true; conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL; - conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; + conf->resetOffset = TMQ_OFFSET__RESET_EARLIEST; conf->hbBgEnable = true; return conf; @@ -318,7 +318,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->resetOffset = TMQ_OFFSET__RESET_NONE; return TMQ_CONF_OK; } else if (strcasecmp(value, "earliest") == 0) { - conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; + conf->resetOffset = TMQ_OFFSET__RESET_EARLIEST; return TMQ_CONF_OK; } else if (strcasecmp(value, "latest") == 0) { conf->resetOffset = TMQ_OFFSET__RESET_LATEST; @@ -809,7 +809,6 @@ void tmqSendHbReq(void* param, void* tmrId) { offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; offRows->offset = pVg->offsetInfo.committedOffset; - tscDebug("report row:%lldd, offset:%" PRId64, offRows->rows, offRows->offset.version); } } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 21721d85a5..722092a043 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -361,11 +361,11 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "withTbName", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, - {.name = "useSnapshot", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, - {.name = "autoCommit", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, - {.name = "autoCommitInterval", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, - {.name = "resetOffsetCfg", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "msg.with.table.name", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, + {.name = "experimental.snapshot.enable", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, + {.name = "enable.auto.commit", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, + {.name = "auto.commit.interval.ms", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "auto.offset.reset", .bytes = TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, }; static const SSysDbTableSchema offsetSchema[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f14375f9bf..40ed29dc99 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7131,15 +7131,15 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { if (pVal->type == TMQ_OFFSET__RESET_NONE) { - snprintf(buf, maxLen, "offset(reset to none)"); - } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEAST) { - snprintf(buf, maxLen, "offset(reset to earlieast)"); + snprintf(buf, maxLen, "none"); + } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEST) { + snprintf(buf, maxLen, "earliest"); } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { - snprintf(buf, maxLen, "offset(reset to latest)"); + snprintf(buf, maxLen, "latest"); } else if (pVal->type == TMQ_OFFSET__LOG) { - snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version); + snprintf(buf, maxLen, "log:%" PRId64, pVal->version); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) { - snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts); + snprintf(buf, maxLen, "snapshot:%" PRId64 "|%" PRId64, pVal->uid, pVal->ts); } else { return TSDB_CODE_INVALID_PARA; } @@ -7157,7 +7157,7 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) { return pLeft->uid == pRight->uid; } else { ASSERT(0); - /*ASSERT(pLeft->type == TMQ_OFFSET__RESET_NONE || pLeft->type == TMQ_OFFSET__RESET_EARLIEAST ||*/ + /*ASSERT(pLeft->type == TMQ_OFFSET__RESET_NONE || pLeft->type == TMQ_OFFSET__RESET_EARLIEST ||*/ /*pLeft->type == TMQ_OFFSET__RESET_LATEST);*/ /*return true;*/ } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4a6a37395d..fc1a30ef5d 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -1197,8 +1197,13 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->autoCommitInterval, false); + char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg}; + tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &pVal); + varDataSetLen(buf, strlen(varDataVal(buf))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->resetOffsetCfg, false); + colDataSetVal(pColInfo, numOfRows, (const char *)buf, false); numOfRows++; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0bb3b7575a..c10564a369 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -559,7 +559,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { } else { dataRsp.rspOffset.version = currentVer; // return current consume offset value } - } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { dataRsp.rspOffset.version = ever; @@ -754,7 +754,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosArrayDestroy(tbUidList); } - taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); +id taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, pHandle->consumerId); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index e67b8906a9..8607fd754e 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -107,7 +107,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand return 0; } else { // no poll occurs in this vnode for this topic, let's seek to the right offset value. - if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { if (pRequest->useSnapshot) { tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", consumerId, pHandle->subKey, vgId); From 1903b815d41d116a1bb8aad209a98839d9a9e6a0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 09:45:06 +0800 Subject: [PATCH 094/122] fix:modify offset description to string --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c10564a369..edefd3ba90 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -754,7 +754,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosArrayDestroy(tbUidList); } -id taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); + taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, pHandle->consumerId); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; From d85fdc5516236cf6b3ac08d67659bf35c9e60e93 Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 12 Jun 2023 10:08:25 +0800 Subject: [PATCH 095/122] docs: markdown format --- docs/en/20-third-party/11-kafka.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index f643583312..d40efc702c 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -27,11 +27,11 @@ TDengine Source Connector is used to read data from TDengine in real-time and se Execute in any directory: -```` +```shell curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz tar xzf kafka_2.13-3.4.0.tgz -C /opt/ ln -s /opt/kafka_2.13-3.4.0 /opt/kafka -```` +``` Then you need to add the `$KAFKA_HOME/bin` directory to the PATH. @@ -181,7 +181,7 @@ meters,location=California.LoSangeles,groupid=3 current=11.3,voltage=221,phase=0 Use kafka-console-producer to write test data to the topic `meters`. -``` +```shell cat test-data.txt | kafka-console-producer.sh --broker-list localhost:9092 --topic meters ``` From ae85e5e0f6649c8d3f990e8d0cd9f6580d9aaf51 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 10:29:33 +0800 Subject: [PATCH 096/122] fix:modify offset description to string --- source/client/src/clientTmq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index cb38996ea1..a989f06d84 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2139,6 +2139,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { return rsp; } } + taosSsleep(2); // sleep 2s for hb to send offset and rows to server int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); From b200fdba5120af4079b2e7dd82865c9e6ee0a87b Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jun 2023 10:44:01 +0800 Subject: [PATCH 097/122] chore: node type --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5f3e402027..ed109df1f9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3108,7 +3108,7 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { if ('\0' == pExpr->userAlias[0]) { strcpy(pExpr->userAlias, pExpr->aliasName); } - if (QUERY_NODE_COLUMN == nodeType(pProject) && + if (QUERY_NODE_FUNCTION == nodeType(pProject) && ((0 == strcasecmp("_wstart", pExpr->userAlias) || 0 == strcasecmp("_wend", pExpr->userAlias) || 0 == strcasecmp("_wduration", pExpr->userAlias)) && '\0' != pExpr->aliasName[0])) { From f5d28cdcd1fb99faa9f558bf22ad6c018c768ddc Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jun 2023 10:49:17 +0800 Subject: [PATCH 098/122] chore: more check --- source/libs/parser/src/parTranslater.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ed109df1f9..a6d010a814 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3108,12 +3108,12 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { if ('\0' == pExpr->userAlias[0]) { strcpy(pExpr->userAlias, pExpr->aliasName); } - if (QUERY_NODE_FUNCTION == nodeType(pProject) && + if (QUERY_NODE_FUNCTION == nodeType(pProject) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pProject)->funcId) && ((0 == strcasecmp("_wstart", pExpr->userAlias) || 0 == strcasecmp("_wend", pExpr->userAlias) || 0 == strcasecmp("_wduration", pExpr->userAlias)) && '\0' != pExpr->aliasName[0])) { - continue; - } + continue; + } sprintf(pExpr->aliasName, "#expr_%d", no++); } return TSDB_CODE_SUCCESS; From 5c5e76f894cdb445968b568e136385d4d304b805 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 11:17:32 +0800 Subject: [PATCH 099/122] fix:modify offset description to string --- source/client/src/clientTmq.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index a989f06d84..a4d7760a7d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -985,6 +985,14 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { } int32_t tmq_unsubscribe(tmq_t* tmq) { + if (tmq->autoCommit) { + int32_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp != 0) { + return rsp; + } + } + taosSsleep(2); // sleep 2s for hb to send offset and rows to server + int32_t rsp; int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); From 16fbcbb67ab20910b4c8b1105355bde12b783a11 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 12 Jun 2023 14:18:06 +0800 Subject: [PATCH 100/122] fix: filter by tbFName in metaFilterTableByHash --- source/dnode/vnode/src/meta/metaTable.c | 9 +++++++-- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index eb169fbdc2..0d0716f2f0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -879,9 +879,13 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { SDecoder dc = {0}; tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &me); + if (me.type != TSDB_SUPER_TABLE) { - int32_t ret = vnodeValidateTableHash(pMeta->pVnode, me.name); - if (TSDB_CODE_VND_HASH_MISMATCH == ret) { + char tbFName[TSDB_TABLE_FNAME_LEN + 1]; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pMeta->pVnode->config.dbname, me.name); + tbFName[TSDB_TABLE_FNAME_LEN] = '\0'; + int32_t ret = vnodeValidateTableHash(pMeta->pVnode, tbFName); + if (ret < 0 && terrno == TSDB_CODE_VND_HASH_MISMATCH) { taosArrayPush(uidList, &me.uid); } } @@ -910,6 +914,7 @@ int32_t metaTrimTables(SMeta *pMeta) { goto end; } + metaInfo("vgId:%d, trim %ld tables", TD_VID(pMeta->pVnode), taosArrayGetSize(tbUids)); metaDropTables(pMeta, tbUids); end: diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index faa4d2fc57..efe82e1783 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -325,7 +325,7 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) { if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) { terrno = TSDB_CODE_VND_HASH_MISMATCH; - return TSDB_CODE_VND_HASH_MISMATCH; + return -1; } return 0; From 170182fecd1b65810edca8752fbc10453604482f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 12 Jun 2023 16:10:54 +0800 Subject: [PATCH 101/122] feat: support fill history with sub request --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 1 + include/libs/function/functionMgt.h | 1 + include/libs/nodes/cmdnodes.h | 22 +- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 2 + include/libs/parser/parser.h | 1 + include/libs/planner/planner.h | 1 + source/client/inc/clientInt.h | 16 ++ source/client/src/clientEnv.c | 42 ++++ source/client/src/clientImpl.c | 193 ++++++++++++++++- source/client/src/clientMain.c | 282 ++++++++++++++++--------- source/common/src/tmsg.c | 5 + source/libs/function/src/functionMgt.c | 2 +- source/libs/nodes/src/nodesUtilFuncs.c | 3 + source/libs/parser/inc/parInt.h | 1 + source/libs/parser/src/parAstParser.c | 4 + source/libs/parser/src/parTranslater.c | 176 +++++++++++++-- source/libs/parser/src/parser.c | 15 +- source/libs/planner/src/planner.c | 6 + 20 files changed, 636 insertions(+), 139 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d78e771fcf..86308984ef 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1975,6 +1975,7 @@ typedef struct { SArray* fillNullCols; // array of SColLocation int64_t deleteMark; int8_t igUpdate; + int64_t lastTs; } SCMCreateStreamReq; typedef struct { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 9300deeb9a..7a7a13b285 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -87,6 +87,7 @@ typedef struct SCatalogReq { bool dNodeRequired; // valid dnode bool svrVerRequired; bool forceUpdate; + bool cloned; } SCatalogReq; typedef struct SMetaRes { diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f0c9cffd0f..55af50e0bc 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -233,6 +233,7 @@ bool fmIsGroupKeyFunc(int32_t funcId); bool fmIsBlockDistFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType); +SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index c8ce9634f5..3a36601b11 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -425,16 +425,18 @@ typedef struct SStreamOptions { } SStreamOptions; typedef struct SCreateStreamStmt { - ENodeType type; - char streamName[TSDB_TABLE_NAME_LEN]; - char targetDbName[TSDB_DB_NAME_LEN]; - char targetTabName[TSDB_TABLE_NAME_LEN]; - bool ignoreExists; - SStreamOptions* pOptions; - SNode* pQuery; - SNodeList* pTags; - SNode* pSubtable; - SNodeList* pCols; + ENodeType type; + char streamName[TSDB_TABLE_NAME_LEN]; + char targetDbName[TSDB_DB_NAME_LEN]; + char targetTabName[TSDB_TABLE_NAME_LEN]; + bool ignoreExists; + SStreamOptions* pOptions; + SNode* pQuery; + SNode* pPrevQuery; + SNodeList* pTags; + SNode* pSubtable; + SNodeList* pCols; + SCMCreateStreamReq* pReq; } SCreateStreamStmt; typedef struct SDropStreamStmt { diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 02459ed951..f44b622cc0 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -617,6 +617,7 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; + void* pPostPlan; } SQueryPlan; const char* dataOrderStr(EDataOrderLevel order); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 12890571f9..f570698395 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -441,7 +441,9 @@ typedef struct SQuery { EQueryExecStage execStage; EQueryExecMode execMode; bool haveResultSet; + SNode* pPrevRoot; SNode* pRoot; + SNode* pPostRoot; int32_t numOfResCols; SSchema* pResSchema; int8_t precision; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 94fb6824d2..f253b47e50 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -74,6 +74,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata const struct SMetaData* pMetaData, SQuery* pQuery); int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery); +int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pResRow); void qDestroyParseContext(SParseContext* pCxt); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 41c0e98084..1b523c0323 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -52,6 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @groupId id of a group of datasource subplans of this @pSubplan // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); +int32_t qContinuePlanPostQuery(void *pPostPlan); void qClearSubplanExecutionNode(SSubplan* pSubplan); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 18891bb932..afa19c674e 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -227,6 +227,12 @@ typedef struct { STaosxRsp rsp; } SMqTaosxRspObj; +typedef struct SReqRelInfo { + uint64_t userRefId; + uint64_t prevRefId; + uint64_t nextRefId; +} SReqRelInfo; + typedef struct SRequestObj { int8_t resType; // query or tmq uint64_t requestId; @@ -254,6 +260,9 @@ typedef struct SRequestObj { uint32_t retry; int64_t allocatorRefId; SQuery* pQuery; + void* pPostPlan; + SReqRelInfo relation; + void* pWrapper; } SRequestObj; typedef struct SSyncQueryParam { @@ -279,6 +288,7 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int64_t reqid); +void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); @@ -368,6 +378,7 @@ typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; SCatalogReq* pCatalogReq; SRequestObj* pRequest; + void* pPlanInfo; } SSqlCallbackWrapper; SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); @@ -382,6 +393,11 @@ int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); bool qnodeRequired(SRequestObj* pRequest); void continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper); +void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code); +void restartAsyncQuery(SRequestObj *pRequest, int32_t code); +int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest); +int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce); +void returnToUser(SRequestObj* pRequest); #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 045642c2c2..4f883ca1f4 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -358,6 +358,44 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } + +void destroySubRequests(SRequestObj *pRequest) { + int32_t reqIdx = -1; + SRequestObj *pReqList[16] = {NULL}; + uint64_t tmpRefId = 0; + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + tmpRefId = pTmp->relation.prevRefId; + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + pReqList[++reqIdx] = pTmp; + releaseRequest(tmpRefId); + } else { + tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, + tmpRefId, pTmp->requestId); + break; + } + } + + for (int32_t i = reqIdx; i >= 0; i--) { + removeRequest(pReqList[i]->self); + } + + tmpRefId = pRequest->relation.nextRefId; + while (tmpRefId) { + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + tmpRefId = pTmp->relation.nextRefId; + removeRequest(pTmp->self); + releaseRequest(pTmp->self); + } else { + tscError("0x%" PRIx64 " is not there", tmpRefId); + break; + } + } +} + + void doDestroyRequest(void *p) { if (NULL == p) { return; @@ -368,10 +406,14 @@ void doDestroyRequest(void *p) { uint64_t reqId = pRequest->requestId; tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); + destroySubRequests(pRequest); + taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); schedulerFreeJob(&pRequest->body.queryJob, 0); + destorySqlCallbackWrapper(pRequest->pWrapper); + taosMemoryFreeClear(pRequest->msgBuf); taosMemoryFreeClear(pRequest->pDb); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5963e419e1..de14266099 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -237,6 +237,16 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, return TSDB_CODE_SUCCESS; } +int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest) { + int32_t code = buildRequest(pRequest->pTscObj->id, sql, strlen(sql), pRequest, pRequest->validateOnly, pNewRequest, 0); + if (TSDB_CODE_SUCCESS == code) { + pRequest->relation.prevRefId = (*pNewRequest)->self; + (*pNewRequest)->relation.nextRefId = pRequest->self; + (*pNewRequest)->relation.userRefId = pRequest->self; + } + return code; +} + int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb) { STscObj* pTscObj = pRequest->pTscObj; @@ -878,6 +888,81 @@ static bool incompletaFileParsing(SNode* pStmt) { return QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pStmt) ? false : ((SVnodeModifyOpStmt*)pStmt)->fileProcessing; } +void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) { + SSqlCallbackWrapper* pWrapper = pRequest->pWrapper; + int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + int64_t analyseStart = taosGetTimestampUs(); + code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, (void**)row); + pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart; + } + if (TSDB_CODE_SUCCESS == code) { + code = qContinuePlanPostQuery(pRequest->pPostPlan); + } + nodesReleaseAllocator(pWrapper->pParseCtx->allocatorId); + + handleQueryAnslyseRes(pWrapper, NULL, code); +} + +void returnToUser(SRequestObj* pRequest) { + if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) { + // return to client + pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); + return; + } + + SRequestObj* pUserReq = acquireRequest(pRequest->relation.userRefId); + if (pUserReq) { + pUserReq->code = pRequest->code; + // return to client + pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code); + releaseRequest(pRequest->relation.userRefId); + return; + } else { + tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + pRequest->relation.userRefId, pRequest->requestId); + } +} + +void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) { + SRequestObj* pRequest = (SRequestObj*)res; + if (pRequest->code) { + returnToUser(pRequest); + return; + } + + TAOS_ROW row = NULL; + if (rowNum > 0) { + row = taos_fetch_row(res); // for single row only now + } + + SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId); + if (pNextReq) { + continuePostSubQuery(pNextReq, row); + releaseRequest(pRequest->relation.nextRefId); + } else { + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + pRequest->relation.nextRefId, pRequest->requestId); + } +} + +void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) { + SRequestObj* pRequest = pWrapper->pRequest; + if (TD_RES_QUERY(pRequest)) { + taosAsyncFetchImpl(pRequest, postSubQueryFetchCb, pWrapper); + return; + } + + SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId); + if (pNextReq) { + continuePostSubQuery(pNextReq, NULL); + releaseRequest(pRequest->relation.nextRefId); + } else { + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + pRequest->relation.nextRefId, pRequest->requestId); + } +} + // todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SSqlCallbackWrapper* pWrapper = param; @@ -912,12 +997,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId); - pRequest->prevCode = code; - schedulerFreeJob(&pRequest->body.queryJob, 0); - qDestroyQuery(pRequest->pQuery); - pRequest->pQuery = NULL; - destorySqlCallbackWrapper(pWrapper); - doAsyncQuery(pRequest, true); + restartAsyncQuery(pRequest, code); return; } @@ -938,10 +1018,15 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { return; } - destorySqlCallbackWrapper(pWrapper); + if (pRequest->relation.nextRefId) { + handlePostSubQuery(pWrapper); + } else { + destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; - // return to client - pRequest->body.queryFp(pRequest->body.param, pRequest, code); + // return to client + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } } SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { @@ -1049,6 +1134,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->requestId); } else { pRequest->body.subplanNum = pDag->numOfSubplans; + TSWAP(pRequest->pPostPlan, pDag->pPostPlan); } pRequest->metric.execStart = taosGetTimestampUs(); @@ -1084,6 +1170,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; if (TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -1103,6 +1190,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.execMode = pQuery->execMode; if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; } if (pQuery->pRoot && !pRequest->inRetry) { @@ -2402,3 +2490,90 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, return pRequest; } + + +static void fetchCallback(void *pResult, void *param, int32_t code) { + SRequestObj *pRequest = (SRequestObj *)param; + + SReqResultInfo *pResultInfo = &pRequest->body.resInfo; + + tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); + + pResultInfo->pData = pResult; + pResultInfo->numOfRows = 0; + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + taosMemoryFreeClear(pResultInfo->pData); + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + return; + } + + if (pRequest->code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pResultInfo->pData); + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + return; + } + + pRequest->code = + setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, true); + if (pRequest->code != TSDB_CODE_SUCCESS) { + pResultInfo->numOfRows = 0; + pRequest->code = code; + tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, + pRequest->requestId); + + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); + } + + pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); +} + +void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param) { + pRequest->body.fetchFp = fp; + pRequest->body.param = param; + + SReqResultInfo *pResultInfo = &pRequest->body.resInfo; + + // this query has no results or error exists, return directly + if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + pResultInfo->numOfRows = 0; + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); + return; + } + + // all data has returned to App already, no need to try again + if (pResultInfo->completed) { + // it is a local executed query, no need to do async fetch + if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { + if (pResultInfo->localResultFetched) { + pResultInfo->numOfRows = 0; + pResultInfo->current = 0; + } else { + pResultInfo->localResultFetched = true; + } + } else { + pResultInfo->numOfRows = 0; + } + + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); + return; + } + + SSchedulerReq req = { + .syncReq = false, + .fetchFp = fetchCallback, + .cbParam = pRequest, + }; + + schedulerFetchRows(pRequest->body.queryJob, &req); +} + + diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 63a4e5d2e5..ca4acf3edf 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -774,8 +774,13 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) { taosArrayDestroy(pCatalogReq->pDbVgroup); taosArrayDestroy(pCatalogReq->pDbCfg); taosArrayDestroy(pCatalogReq->pDbInfo); - taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); - taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); + if (pCatalogReq->cloned) { + taosArrayDestroy(pCatalogReq->pTableMeta); + taosArrayDestroy(pCatalogReq->pTableHash); + } else { + taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); + taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); + } taosArrayDestroy(pCatalogReq->pUdf); taosArrayDestroy(pCatalogReq->pIndex); taosArrayDestroy(pCatalogReq->pUser); @@ -794,26 +799,108 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) { taosMemoryFree(pWrapper); } +void destroyCtxInRequest(SRequestObj* pRequest) { + schedulerFreeJob(&pRequest->body.queryJob, 0); + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = NULL; + destorySqlCallbackWrapper(pRequest->pWrapper); + pRequest->pWrapper = NULL; +} + + static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) { SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; SRequestObj *pRequest = pWrapper->pRequest; SQuery *pQuery = pRequest->pQuery; - int64_t analyseStart = taosGetTimestampUs(); - pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); - if (code == TSDB_CODE_SUCCESS) { + int64_t analyseStart = taosGetTimestampUs(); + pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; + + if (TSDB_CODE_SUCCESS == code) { code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); + } + + pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart; + + handleQueryAnslyseRes(pWrapper, pResultMeta, code); +} + +int32_t cloneCatalogReq(SCatalogReq* * ppTarget, SCatalogReq* pSrc) { + int32_t code = TSDB_CODE_SUCCESS; + SCatalogReq* pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq)); + if (pTarget == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTarget->pDbVgroup = taosArrayDup(pSrc->pDbVgroup, NULL); + pTarget->pDbCfg = taosArrayDup(pSrc->pDbCfg, NULL); + pTarget->pDbInfo = taosArrayDup(pSrc->pDbInfo, NULL); + pTarget->pTableMeta = taosArrayDup(pSrc->pTableMeta, NULL); + pTarget->pTableHash = taosArrayDup(pSrc->pTableHash, NULL); + pTarget->pUdf = taosArrayDup(pSrc->pUdf, NULL); + pTarget->pIndex = taosArrayDup(pSrc->pIndex, NULL); + pTarget->pUser = taosArrayDup(pSrc->pUser, NULL); + pTarget->pTableIndex = taosArrayDup(pSrc->pTableIndex, NULL); + pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL); + pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL); + pTarget->qNodeRequired = pSrc->qNodeRequired; + pTarget->dNodeRequired = pSrc->dNodeRequired; + pTarget->svrVerRequired = pSrc->svrVerRequired; + pTarget->forceUpdate = pSrc->forceUpdate; + pTarget->cloned = true; + + *ppTarget = pTarget; + } + + return code; +} + + +void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode* pRoot) { + SRequestObj* pNewRequest = NULL; + SSqlCallbackWrapper* pNewWrapper = NULL; + int32_t code = buildPreviousRequest(pWrapper->pRequest, "", &pNewRequest); + if (code) { + handleQueryAnslyseRes(pWrapper, pResultMeta, code); + return; + } + + pNewRequest->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pNewRequest->pQuery) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pNewRequest->pQuery->pRoot = pRoot; + pRoot = NULL; + pNewRequest->pQuery->execStage = QUERY_EXEC_STAGE_ANALYSE; + } + if (TSDB_CODE_SUCCESS == code) { + code = prepareAndParseSqlSyntax(&pNewWrapper, pNewRequest, false); + } + if (TSDB_CODE_SUCCESS == code) { + code = cloneCatalogReq(&pNewWrapper->pCatalogReq, pWrapper->pCatalogReq); + } + doAsyncQueryFromAnalyse(pResultMeta, pNewWrapper, code); + nodesDestroyNode(pRoot); +} + +void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code) { + SRequestObj *pRequest = pWrapper->pRequest; + SQuery *pQuery = pRequest->pQuery; + + if (code == TSDB_CODE_SUCCESS && pQuery->pPrevRoot) { + SNode* prevRoot = pQuery->pPrevRoot; + pQuery->pPrevRoot = NULL; + handleSubQueryFromAnalyse(pWrapper, pResultMeta, prevRoot); + return; + } + + if (code == TSDB_CODE_SUCCESS) { pRequest->stableQuery = pQuery->stableQuery; if (pQuery->pRoot) { pRequest->stmtType = pQuery->pRoot->type; } - } - pRequest->metric.analyseCostUs = taosGetTimestampUs() - analyseStart; - - if (code == TSDB_CODE_SUCCESS) { if (pQuery->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); setResPrecision(&pRequest->body.resInfo, pQuery->precision); @@ -826,14 +913,14 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t launchAsyncQuery(pRequest, pQuery, pResultMeta, pWrapper); } else { destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); - pRequest->prevCode = code; - doAsyncQuery(pRequest, true); + restartAsyncQuery(pRequest, code); return; } @@ -841,7 +928,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); pRequest->code = code; - pRequest->body.queryFp(pRequest->body.param, pRequest, code); + returnToUser(pRequest); } } @@ -904,6 +991,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; terrno = code; pRequest->code = code; pRequest->body.queryFp(pRequest->body.param, pRequest, code); @@ -920,6 +1008,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; terrno = code; pRequest->code = code; pRequest->body.queryFp(pRequest->body.param, pRequest, code); @@ -967,27 +1056,16 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { return TSDB_CODE_SUCCESS; } -void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { +int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce) { + int32_t code = TSDB_CODE_SUCCESS; STscObj *pTscObj = pRequest->pTscObj; - SSqlCallbackWrapper *pWrapper = NULL; - int32_t code = TSDB_CODE_SUCCESS; - - if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { - code = pRequest->prevCode; - terrno = code; - pRequest->code = code; - tscDebug("call sync query cb with code: %s", tstrerror(code)); - pRequest->body.queryFp(pRequest->body.param, pRequest, code); - return; - } - - if (TSDB_CODE_SUCCESS == code) { - pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - pWrapper->pRequest = pRequest; - } + SSqlCallbackWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); + if (pWrapper == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pWrapper->pRequest = pRequest; + pRequest->pWrapper = pWrapper; + *ppWrapper = pWrapper; } if (TSDB_CODE_SUCCESS == code) { @@ -999,7 +1077,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pWrapper->pParseCtx->pCatalog); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && NULL == pRequest->pQuery) { int64_t syntaxStart = taosGetTimestampUs(); pWrapper->pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReq)); @@ -1014,6 +1092,27 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { pRequest->metric.parseCostUs += taosGetTimestampUs() - syntaxStart; } + return code; +} + + +void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { + SSqlCallbackWrapper *pWrapper = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { + code = pRequest->prevCode; + terrno = code; + pRequest->code = code; + tscDebug("call sync query cb with code: %s", tstrerror(code)); + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + return; + } + + if (TSDB_CODE_SUCCESS == code) { + code = prepareAndParseSqlSyntax(&pWrapper, pRequest, updateMetaForce); + } + if (TSDB_CODE_SUCCESS == code) { pRequest->stmtType = pRequest->pQuery->pRoot->type; code = phaseAsyncQuery(pWrapper); @@ -1023,6 +1122,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; @@ -1040,48 +1140,57 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } } -static void fetchCallback(void *pResult, void *param, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)param; - - SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - - tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); - - pResultInfo->pData = pResult; - pResultInfo->numOfRows = 0; - - if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - taosMemoryFreeClear(pResultInfo->pData); - pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); - return; +void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { + int32_t reqIdx = 0; + SRequestObj *pReqList[16] = {NULL}; + SRequestObj *pUserReq = NULL; + pReqList[0] = pRequest; + uint64_t tmpRefId = 0; + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + tmpRefId = pTmp->relation.prevRefId; + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + pReqList[++reqIdx] = pTmp; + releaseRequest(tmpRefId); + } else { + tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, + tmpRefId, pTmp->requestId); + break; + } } - if (pRequest->code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pResultInfo->pData); - pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); - return; + tmpRefId = pRequest->relation.nextRefId; + while (tmpRefId) { + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + tmpRefId = pTmp->relation.nextRefId; + removeRequest(pTmp->self); + releaseRequest(pTmp->self); + } else { + tscError("0x%" PRIx64 " is not there", tmpRefId); + break; + } } - pRequest->code = - setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, true); - if (pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - pRequest->code = code; - tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); + for (int32_t i = reqIdx; i >= 0; i--) { + destroyCtxInRequest(pReqList[i]); + if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) { + pUserReq = pReqList[i]; + } else { + removeRequest(pReqList[i]->self); + } + } + + if (pUserReq) { + pUserReq->prevCode = code; + memset(&pUserReq->relation, 0, sizeof(pUserReq->relation)); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, - pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, - pRequest->requestId); - - STscObj *pTscObj = pRequest->pTscObj; - SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; - atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); + tscError("user req is missing"); + return; } - pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); + doAsyncQuery(pUserReq, true); } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { @@ -1095,43 +1204,8 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } SRequestObj *pRequest = res; - pRequest->body.fetchFp = fp; - pRequest->body.param = param; - SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - - // this query has no results or error exists, return directly - if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); - return; - } - - // all data has returned to App already, no need to try again - if (pResultInfo->completed) { - // it is a local executed query, no need to do async fetch - if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { - if (pResultInfo->localResultFetched) { - pResultInfo->numOfRows = 0; - pResultInfo->current = 0; - } else { - pResultInfo->localResultFetched = true; - } - } else { - pResultInfo->numOfRows = 0; - } - - pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); - return; - } - - SSchedulerReq req = { - .syncReq = false, - .fetchFp = fetchCallback, - .cbParam = pRequest, - }; - - schedulerFetchRows(pRequest->body.queryJob, &req); + taosAsyncFetchImpl(pRequest, fp, param); } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index ac035e0a2b..cd874dad50 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6122,6 +6122,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS } if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; if (tEncodeI8(&encoder, pReq->igUpdate) < 0) return -1; + if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1; tEndEncode(&encoder); @@ -6207,6 +6208,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igUpdate) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1; tEndDecode(&decoder); @@ -6273,6 +6275,9 @@ int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStrea } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { + if (NULL == pReq) { + return; + } taosArrayDestroy(pReq->pTags); taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 18f6e8050b..327bc7da71 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -344,7 +344,7 @@ static int32_t getFuncInfo(SFunctionNode* pFunc) { return fmGetFuncInfo(pFunc, msg, sizeof(msg)); } -static SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { +SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { return NULL; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 39e288f694..ee2bc7e442 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -953,6 +953,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pQuery); nodesDestroyList(pStmt->pTags); nodesDestroyNode(pStmt->pSubtable); + tFreeSCMCreateStreamReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); break; } case QUERY_NODE_DROP_STREAM_STMT: // no pointer field @@ -1052,6 +1054,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_QUERY: { SQuery* pQuery = (SQuery*)pNode; nodesDestroyNode(pQuery->pRoot); + nodesDestroyNode(pQuery->pPostRoot); taosMemoryFreeClear(pQuery->pResSchema); if (NULL != pQuery->pCmdMsg) { taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 66aec272d7..d79aa84bb8 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -34,6 +34,7 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); +int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 801d43e2a4..fdec9cba79 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -384,6 +384,10 @@ static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateS if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } + if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + code = reserveDbCfgForLastRow(pCxt, pSelect->pFromTable); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d1c3da0041..905b32a5f3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -53,6 +53,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; + SNode* pPrevRoot; + SNode* pPostRoot; } STranslateContext; typedef struct SBuildTopicContext { @@ -276,6 +278,10 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode); static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal); +static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt); +static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery); +static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); +static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; } @@ -6730,6 +6736,54 @@ static int32_t translateStreamTargetTable(STranslateContext* pCxt, SCreateStream return code; } +static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery) { + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == col) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + strcpy(col->tableAlias, pTable); + strcpy(col->colName, pMeta->schema[0].name); + SNodeList* pParamterList = nodesMakeList(); + if (NULL == pParamterList) { + nodesDestroyNode((SNode *)col); + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesListStrictAppend(pParamterList, (SNode *)col); + if (code) { + nodesDestroyNode((SNode *)col); + nodesDestroyList(pParamterList); + return code; + } + + SNode* pFunc = (SNode*)createFunction("last", pParamterList); + if (NULL == pFunc) { + nodesDestroyList(pParamterList); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pProjectionList = nodesMakeList(); + if (NULL == pProjectionList) { + nodesDestroyList(pParamterList); + return TSDB_CODE_OUT_OF_MEMORY; + } + code = nodesListStrictAppend(pProjectionList, pFunc); + if (code) { + nodesDestroyNode(pFunc); + nodesDestroyList(pProjectionList); + return code; + } + + code = createSimpleSelectStmtFromProjList(pDb, pTable, pProjectionList, (SSelectStmt **)pQuery); + if (code) { + nodesDestroyList(pProjectionList); + return code; + } + + return code; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; STableMeta* pMeta = NULL; @@ -6756,6 +6810,18 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } + if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) { + SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable); + code = createLastTsSelectStmt(pTable->table.dbName, pTable->table.tableName, pTable->pMeta, &pStmt->pPrevQuery); +/* + if (TSDB_CODE_SUCCESS == code) { + STranslateContext cxt = {0}; + int32_t code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt); + code = translateQuery(&cxt, pStmt->pPrevQuery); + destroyTranslateContext(&cxt); + } +*/ + } taosMemoryFree(pMeta); return code; } @@ -6822,13 +6888,80 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* code = buildCreateStreamReq(pCxt, pStmt, &createReq); } if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, &createReq); + if (NULL == pStmt->pPrevQuery) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, &createReq); + } else { + pStmt->pReq = taosMemoryMalloc(sizeof(createReq)); + if (NULL == pStmt->pReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(pStmt->pReq, &createReq, sizeof(createReq)); + memset(&createReq, 0, sizeof(createReq)); + TSWAP(pCxt->pPrevRoot, pStmt->pPrevQuery); + } + } } tFreeSCMCreateStreamReq(&createReq); return code; } +int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* pInterval) { + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery)) { + return code; + } + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow)) { + return code; + } + + SIntervalWindowNode* pWindow = (SIntervalWindowNode*)pSelect->pWindow; + pInterval->interval = ((SValueNode*)pWindow->pInterval)->datum.i; + pInterval->intervalUnit = ((SValueNode*)pWindow->pInterval)->unit; + pInterval->offset = (NULL != pWindow->pOffset ? ((SValueNode*)pWindow->pOffset)->datum.i : 0); + pInterval->sliding = (NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->datum.i : pInterval->interval); + pInterval->slidingUnit = + (NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->unit : pInterval->intervalUnit); + pInterval->precision = ((SColumnNode*)pWindow->pCol)->node.resType.precision; + + return code; +} + +int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) { + SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot; + STranslateContext cxt = {0}; + SInterval interval = {0}; + + if (NULL == pResRow || NULL == pResRow[0]) { + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + + int32_t code = initTranslateContext(pParseCxt, NULL, &cxt); + if (TSDB_CODE_SUCCESS == code) { + code = buildIntervalForCreateStream(pStmt, &interval); + } + if (TSDB_CODE_SUCCESS == code) { + if (interval.interval > 0) { + pStmt->pReq->lastTs = taosTimeTruncate(*(int64_t*)pResRow[0], &interval); + } else { + pStmt->pReq->lastTs = *(int64_t*)pResRow[0]; + } + code = buildCmdMsg(&cxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, pStmt->pReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = setQuery(&cxt, pQuery); + } + setRefreshMate(&cxt, pQuery); + destroyTranslateContext(&cxt); + + tFreeSCMCreateStreamReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); + + return code; +} + + static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) { SMDropStreamReq dropReq = {0}; SName name; @@ -7509,8 +7642,7 @@ static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) { return pProjections; } -static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32_t numOfProjs, - const char* const pProjCol[], SSelectStmt** pStmt) { +static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7526,27 +7658,38 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32 snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); pSelect->pFromTable = (SNode*)pRealTable; - - if (numOfProjs >= 0) { - pSelect->pProjectionList = createProjectCols(numOfProjs, pProjCol); - if (NULL == pSelect->pProjectionList) { - nodesDestroyNode((SNode*)pSelect); - return TSDB_CODE_OUT_OF_MEMORY; - } - } + pSelect->pProjectionList = pProjectionList; *pStmt = pSelect; return TSDB_CODE_SUCCESS; } + +static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs, + const char* const pProjCol[], SSelectStmt** pStmt) { + SNodeList* pProjectionList = NULL; + if (numOfProjs >= 0) { + pProjectionList = createProjectCols(numOfProjs, pProjCol); + if (NULL == pProjectionList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); +} + +static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { + return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); +} + static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) { const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET]; - return createSimpleSelectStmt(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt); + return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt); } static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) { - return createSimpleSelectStmt(pStmt->dbName, pStmt->tableName, 0, NULL, pOutput); + return createSimpleSelectStmtFromCols(pStmt->dbName, pStmt->tableName, 0, NULL, pOutput); } static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SNode* pRight, SNode** pOp) { @@ -7680,7 +7823,7 @@ static int32_t createShowTableTagsProjections(SNodeList** pProjections, SNodeLis static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) { SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot; SSelectStmt* pSelect = NULL; - int32_t code = createSimpleSelectStmt(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, + int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect); if (TSDB_CODE_SUCCESS == code) { code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags); @@ -9005,6 +9148,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { } break; default: + pQuery->haveResultSet = false; pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); @@ -9039,6 +9183,10 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe if (TSDB_CODE_SUCCESS == code) { code = translateQuery(&cxt, pQuery->pRoot); } + if (TSDB_CODE_SUCCESS == code && (cxt.pPrevRoot || cxt.pPostRoot)) { + pQuery->pPrevRoot = cxt.pPrevRoot; + pQuery->pPostRoot = cxt.pPostRoot; + } if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 28d116c381..cbddaf8115 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -204,7 +204,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata const struct SMetaData* pMetaData, SQuery* pQuery) { SParseMetaCache metaCache = {0}; int32_t code = nodesAcquireAllocator(pCxt->allocatorId); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && pCatalogReq) { code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache); } if (TSDB_CODE_SUCCESS == code) { @@ -221,6 +221,19 @@ int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, return parseInsertSql(pCxt, &pQuery, pCatalogReq, pMetaData); } +int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pResRow) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pQuery->pRoot)) { + case QUERY_NODE_CREATE_STREAM_STMT: + code = translatePostCreateStream(pCxt, pQuery, pResRow); + break; + default: + break; + } + + return code; +} + void qDestroyParseContext(SParseContext* pCxt) { if (NULL == pCxt) { return; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 58b8e53478..2fcc8510d4 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -97,6 +97,12 @@ static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDown return TSDB_CODE_SUCCESS; } +int32_t qContinuePlanPostQuery(void *pPostPlan) { + //TODO + return TSDB_CODE_SUCCESS; +} + + int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); From fe155898fe79cb3e02f5029ea29c5f2cea6a1f2c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 16:34:53 +0800 Subject: [PATCH 102/122] fix:modify offset description to string --- source/client/src/clientTmq.c | 33 +++++++----- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 5 +- source/dnode/mnode/impl/src/mndDef.c | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 52 +++++++++++++------ .../system-test/7-tmq/checkOffsetRowParams.py | 6 +-- 6 files changed, 63 insertions(+), 36 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index a4d7760a7d..26887e2ade 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -99,6 +99,7 @@ struct tmq_t { // poll info int64_t pollCnt; int64_t totalRows; + bool needReportOffsetRows; // timer tmr_h hbLiveTimer; @@ -796,20 +797,24 @@ void tmqSendHbReq(void* param, void* tmrId) { SMqHbReq req = {0}; req.consumerId = tmq->consumerId; req.epoch = tmq->epoch; - req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); - for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){ - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - TopicOffsetRows* data = taosArrayReserve(req.topics, 1); - strcpy(data->topicName, pTopic->topicName); - data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); - for(int j = 0; j < numOfVgroups; j++){ - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); - offRows->vgId = pVg->vgId; - offRows->rows = pVg->numOfRows; - offRows->offset = pVg->offsetInfo.committedOffset; + if(tmq->needReportOffsetRows){ + req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); + for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){ + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + TopicOffsetRows* data = taosArrayReserve(req.topics, 1); + strcpy(data->topicName, pTopic->topicName); + data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); + for(int j = 0; j < numOfVgroups; j++){ + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + offRows->vgId = pVg->vgId; + offRows->rows = pVg->numOfRows; + offRows->offset = pVg->offsetInfo.committedOffset; + tscDebug("report offset: %d", offRows->offset.type); + } } + tmq->needReportOffsetRows = false; } int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); @@ -1087,6 +1092,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->status = TMQ_CONSUMER_STATUS__INIT; pTmq->pollCnt = 0; pTmq->epoch = 0; + pTmq->needReportOffsetRows = true; // set conf strcpy(pTmq->clientId, conf->clientId); @@ -2449,6 +2455,7 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { // if no more waiting rsp pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); taosMemoryFree(pParamSet); + tmq->needReportOffsetRows = true; taosReleaseRef(tmqMgmt.rsetId, refId); return 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 7f8f6a48fa..d975eb1cd1 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -163,7 +163,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_HB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index fc1a30ef5d..b5254dbb88 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -431,13 +431,14 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { mDebug("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); - taosRLockLatch(&pSub->lock); + taosWLockLatch(&pSub->lock); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); if(pConsumerEp){ + taosArrayDestroy(pConsumerEp->offsetRows); pConsumerEp->offsetRows = data->offsetRows; data->offsetRows = NULL; } - taosRUnLockLatch(&pSub->lock); + taosWUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b6fb96882d..7f79408e8c 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -519,6 +519,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp)); } pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp); + pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL); memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); return pSubNew; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index cadd1f53f0..e100f91aef 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -451,24 +451,42 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pOutput->rebVgs, pRebOutput); if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows - if(pSub){ - taosRLockLatch(&pSub->lock); - if(pOutput->pSub->offsetRows == NULL){ - pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); - }else{ - taosArrayClear(pOutput->pSub->offsetRows); - } - pIter = NULL; - while(1){ - pIter = taosHashIterate(pSub->consumerHash, pIter); - if (pIter == NULL) break; - SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - taosArrayAddAll(pOutput->pSub->offsetRows, pConsumerEp->offsetRows); - } - taosRUnLockLatch(&pSub->lock); - mndReleaseSubscribe(pMnode, pSub); + } + } + + if(taosHashGetSize(pOutput->pSub->consumerHash) == 0) { // if all consumer is removed + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows + if (pSub) { + taosRLockLatch(&pSub->lock); + bool init = false; + if (pOutput->pSub->offsetRows == NULL) { + pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); + init = true; } + pIter = NULL; + while (1) { + pIter = taosHashIterate(pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + if (init) { + taosArrayAddAll(pOutput->pSub->offsetRows, pConsumerEp->offsetRows); + mDebug("pSub->offsetRows is init"); + } else { + for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) { + OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j); + for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) { + OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i); + if (d1->vgId == d2->vgId) { + d2->rows += d1->rows; + d2->offset = d1->offset; + mDebug("pSub->offsetRows add vgId:%d, after:%lld, before:%lld", d2->vgId, d2->rows, d1->rows); + } + } + } + } + } + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); } } diff --git a/tests/system-test/7-tmq/checkOffsetRowParams.py b/tests/system-test/7-tmq/checkOffsetRowParams.py index 8e5ff63f60..17c80c68bf 100644 --- a/tests/system-test/7-tmq/checkOffsetRowParams.py +++ b/tests/system-test/7-tmq/checkOffsetRowParams.py @@ -227,12 +227,11 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 10 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - time.sleep(2) tdLog.info("start show subscriptions 1") while(1): tdSql.query("show subscriptions") @@ -240,10 +239,11 @@ class TDTestCase: tdLog.info("sleep") time.sleep(1) elif (tdSql.queryResult[0][4] != None): - tdSql.checkData(0, 4, "offset(reset to earlieast)") + # tdSql.checkData(0, 4, "earliest") tdSql.checkData(0, 5, 0) break + time.sleep(2) tdLog.info("start insert data") self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) self.insert_data(tdSql,\ From 4cd039c38ea372f680f6c1c3fc2dabc218a13d2e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jun 2023 16:47:03 +0800 Subject: [PATCH 103/122] chore: compare funcNode without aliasName --- source/libs/nodes/src/nodesUtilFuncs.c | 6 +++--- source/libs/parser/src/parTranslater.c | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 39e288f694..830092bbf2 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1953,9 +1953,9 @@ static uint32_t funcNodeHash(const char* pKey, uint32_t len) { } static int32_t funcNodeEqual(const void* pLeft, const void* pRight, size_t len) { - if (0 != strcmp((*(const SExprNode**)pLeft)->aliasName, (*(const SExprNode**)pRight)->aliasName)) { - return 1; - } + // if (0 != strcmp((*(const SExprNode**)pLeft)->aliasName, (*(const SExprNode**)pRight)->aliasName)) { + // return 1; + // } return nodesEqualNode(*(const SNode**)pLeft, *(const SNode**)pRight) ? 0 : 1; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a6d010a814..3b94bae05c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3108,12 +3108,6 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { if ('\0' == pExpr->userAlias[0]) { strcpy(pExpr->userAlias, pExpr->aliasName); } - if (QUERY_NODE_FUNCTION == nodeType(pProject) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pProject)->funcId) && - ((0 == strcasecmp("_wstart", pExpr->userAlias) || 0 == strcasecmp("_wend", pExpr->userAlias) || - 0 == strcasecmp("_wduration", pExpr->userAlias)) && - '\0' != pExpr->aliasName[0])) { - continue; - } sprintf(pExpr->aliasName, "#expr_%d", no++); } return TSDB_CODE_SUCCESS; From 59b8cfbc7da720a63c9cce6ba59c4d95444f4577 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 12 Jun 2023 17:14:41 +0800 Subject: [PATCH 104/122] enh: add sub_query in systable and kill subquery --- include/common/tmsg.h | 1 + source/client/inc/clientInt.h | 2 + source/client/src/clientEnv.c | 62 +++++++++++++++++++++ source/client/src/clientHb.c | 1 + source/client/src/clientImpl.c | 1 + source/client/src/clientMain.c | 15 +---- source/common/src/systable.c | 1 + source/common/src/tmsg.c | 2 + source/dnode/mnode/impl/src/mndProfile.c | 3 + source/libs/executor/src/exchangeoperator.c | 2 + 10 files changed, 78 insertions(+), 12 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 86308984ef..5d9161b5a3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2488,6 +2488,7 @@ typedef struct { int64_t stime; // timestamp precision ms int64_t reqRid; bool stableQuery; + bool isSubQuery; char fqdn[TSDB_FQDN_LEN]; int32_t subPlanNum; SArray* subDesc; // SArray diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index afa19c674e..fa444779f3 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -256,6 +256,7 @@ typedef struct SRequestObj { bool validateOnly; // todo refactor bool killed; bool inRetry; + bool isSubReq; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; int64_t allocatorRefId; @@ -398,6 +399,7 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code); int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest); int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce); void returnToUser(SRequestObj* pRequest); +void stopAllQueries(SRequestObj *pRequest); #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 4f883ca1f4..c64bbfbdb6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -363,6 +363,11 @@ void destroySubRequests(SRequestObj *pRequest) { int32_t reqIdx = -1; SRequestObj *pReqList[16] = {NULL}; uint64_t tmpRefId = 0; + + if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) { + return; + } + SRequestObj* pTmp = pRequest; while (pTmp->relation.prevRefId) { tmpRefId = pTmp->relation.prevRefId; @@ -454,6 +459,63 @@ void destroyRequest(SRequestObj *pRequest) { removeRequest(pRequest->self); } +void taosStopQueryImpl(SRequestObj *pRequest) { + pRequest->killed = true; + + // It is not a query, no need to stop. + if (NULL == pRequest->pQuery || QUERY_EXEC_MODE_SCHEDULE != pRequest->pQuery->execMode) { + tscDebug("request 0x%" PRIx64 " no need to be killed since not query", pRequest->requestId); + return; + } + + schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED); + tscDebug("request %" PRIx64 " killed", pRequest->requestId); +} + +void stopAllQueries(SRequestObj *pRequest) { + int32_t reqIdx = -1; + SRequestObj *pReqList[16] = {NULL}; + uint64_t tmpRefId = 0; + + if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) { + return; + } + + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + tmpRefId = pTmp->relation.prevRefId; + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + pReqList[++reqIdx] = pTmp; + releaseRequest(tmpRefId); + } else { + tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, + tmpRefId, pTmp->requestId); + break; + } + } + + for (int32_t i = reqIdx; i >= 0; i--) { + taosStopQueryImpl(pReqList[i]); + } + + taosStopQueryImpl(pRequest); + + tmpRefId = pRequest->relation.nextRefId; + while (tmpRefId) { + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + tmpRefId = pTmp->relation.nextRefId; + taosStopQueryImpl(pTmp); + releaseRequest(pTmp->self); + } else { + tscError("0x%" PRIx64 " is not there", tmpRefId); + break; + } + } +} + + void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); } static void *tscCrashReportThreadFp(void *param) { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2dddfec2bd..cbfa48b322 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -464,6 +464,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.useconds = now - pRequest->metric.start; desc.reqRid = pRequest->self; desc.stableQuery = pRequest->stableQuery; + desc.isSubQuery = pRequest->isSubReq; taosGetFqdn(desc.fqdn); desc.subPlanNum = pRequest->body.subplanNum; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index de14266099..2a73156e8a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -243,6 +243,7 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj pRequest->relation.prevRefId = (*pNewRequest)->self; (*pNewRequest)->relation.nextRefId = pRequest->self; (*pNewRequest)->relation.userRefId = pRequest->self; + (*pNewRequest)->isSubReq = true; } return code; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ca4acf3edf..7573fd5968 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -563,22 +563,13 @@ int taos_select_db(TAOS *taos, const char *db) { return code; } + void taos_stop_query(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return; } - SRequestObj *pRequest = (SRequestObj *)res; - pRequest->killed = true; - - // It is not a query, no need to stop. - if (NULL == pRequest->pQuery || QUERY_EXEC_MODE_SCHEDULE != pRequest->pQuery->execMode) { - tscDebug("request 0x%" PRIx64 " no need to be killed since not query", pRequest->requestId); - return; - } - - schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED); - tscDebug("request %" PRIx64 " killed", pRequest->requestId); + stopAllQueries((SRequestObj*)res); } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { @@ -860,7 +851,7 @@ int32_t cloneCatalogReq(SCatalogReq* * ppTarget, SCatalogReq* pSrc) { void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode* pRoot) { SRequestObj* pNewRequest = NULL; SSqlCallbackWrapper* pNewWrapper = NULL; - int32_t code = buildPreviousRequest(pWrapper->pRequest, "", &pNewRequest); + int32_t code = buildPreviousRequest(pWrapper->pRequest, pWrapper->pRequest->sqlstr, &pNewRequest); if (code) { handleQueryAnslyseRes(pWrapper, pResultMeta, code); return; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index db0cc78de6..f751a9aa9b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -381,6 +381,7 @@ static const SSysDbTableSchema querySchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "exec_usec", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = false}, + {.name = "sub_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = false}, {.name = "sub_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "sub_status", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cd874dad50..c7db62cdec 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -224,6 +224,7 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR if (tEncodeI64(pEncoder, desc->stime) < 0) return -1; if (tEncodeI64(pEncoder, desc->reqRid) < 0) return -1; if (tEncodeI8(pEncoder, desc->stableQuery) < 0) return -1; + if (tEncodeI8(pEncoder, desc->isSubQuery) < 0) return -1; if (tEncodeCStr(pEncoder, desc->fqdn) < 0) return -1; if (tEncodeI32(pEncoder, desc->subPlanNum) < 0) return -1; @@ -291,6 +292,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) if (tDecodeI64(pDecoder, &desc.stime) < 0) return -1; if (tDecodeI64(pDecoder, &desc.reqRid) < 0) return -1; if (tDecodeI8(pDecoder, (int8_t *)&desc.stableQuery) < 0) return -1; + if (tDecodeI8(pDecoder, (int8_t *)&desc.isSubQuery) < 0) return -1; if (tDecodeCStrTo(pDecoder, desc.fqdn) < 0) return -1; if (tDecodeI32(pDecoder, &desc.subPlanNum) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index a1d815189c..460e75b422 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -834,6 +834,9 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->isSubQuery, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 94041140d4..b2d1790faf 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -204,6 +204,8 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + taosSsleep(5); + if (pOperator->status == OP_EXEC_DONE) { return NULL; } From a723fc37c36c95f06ee053fbfc3d7a5025f9e89b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 17:18:06 +0800 Subject: [PATCH 105/122] fix:modify offset description to string --- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e100f91aef..06bbb4798f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -470,7 +470,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; if (init) { taosArrayAddAll(pOutput->pSub->offsetRows, pConsumerEp->offsetRows); - mDebug("pSub->offsetRows is init"); +// mDebug("pSub->offsetRows is init"); } else { for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) { OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j); @@ -479,7 +479,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (d1->vgId == d2->vgId) { d2->rows += d1->rows; d2->offset = d1->offset; - mDebug("pSub->offsetRows add vgId:%d, after:%lld, before:%lld", d2->vgId, d2->rows, d1->rows); +// mDebug("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows); } } } From 7adea3570641953e54bd443276d4b08679986915 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 12 Jun 2023 17:18:38 +0800 Subject: [PATCH 106/122] fix: remove debug info --- source/libs/executor/src/exchangeoperator.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index b2d1790faf..94041140d4 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -204,8 +204,6 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - taosSsleep(5); - if (pOperator->status == OP_EXEC_DONE) { return NULL; } From 1cbe7c8f29f8e7d30dda69cf0b62d6d539e13339 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Jun 2023 19:16:23 +0800 Subject: [PATCH 107/122] fix:column length for ins_columns --- tests/system-test/2-query/odbc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index 5241406b65..b6d2ab2a3f 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,8 +22,8 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - # enterprise version: 288, community version: 280 - tdSql.checkData(0, 0, 288) + # enterprise version: 295, community version: 285 + tdSql.checkData(0, 0, 295) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) From fb1788160939a506ef79488abb441b7ce9720b40 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Jun 2023 09:18:57 +0800 Subject: [PATCH 108/122] fix:test cases --- tests/system-test/7-tmq/tmqCommon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 97f38f01b6..6b633fa193 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -171,7 +171,7 @@ class TMQCom: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return From fb59ec4591d144185b367675210b3409e4240091 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Jun 2023 09:32:54 +0800 Subject: [PATCH 109/122] feat:move plan msg from vginfo to subscription info --- source/dnode/mnode/impl/inc/mndDef.h | 19 ++++----- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- source/dnode/mnode/impl/src/mndDef.c | 46 +++++++++++----------- source/dnode/mnode/impl/src/mndScheduler.c | 28 ++++++------- source/dnode/mnode/impl/src/mndSubscribe.c | 41 +++++++++++++++---- 5 files changed, 81 insertions(+), 55 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 82b714e6eb..38380dee95 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -559,12 +559,12 @@ void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer typedef struct { int32_t vgId; - char* qmsg; // SubPlanToString +// char* qmsg; // SubPlanToString SEpSet epSet; } SMqVgEp; -SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); -void tDeleteSMqVgEp(SMqVgEp* pVgEp); +//SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); +//void tDeleteSMqVgEp(SMqVgEp* pVgEp); int32_t tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp); void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp); @@ -589,6 +589,7 @@ typedef struct { SHashObj* consumerHash; // consumerId -> SMqConsumerEp SArray* unassignedVgs; // SArray char dbName[TSDB_DB_FNAME_LEN]; + char* qmsg; // SubPlanToString } SMqSubscribeObj; SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); @@ -687,12 +688,12 @@ int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver); void tFreeStreamObj(SStreamObj* pObj); -typedef struct { - char streamName[TSDB_STREAM_FNAME_LEN]; - int64_t uid; - int64_t streamUid; - SArray* childInfo; // SArray -} SStreamCheckpointObj; +//typedef struct { +// char streamName[TSDB_STREAM_FNAME_LEN]; +// int64_t uid; +// int64_t streamUid; +// SArray* childInfo; // SArray +//} SStreamCheckpointObj; #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 117c1082a5..f530128572 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -644,7 +644,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SCMSubscribeReq subscribe = {0}; tDeserializeSCMSubscribeReq(msgStr, &subscribe); - uint64_t consumerId = subscribe.consumerId; + int64_t consumerId = subscribe.consumerId; char *cgroup = subscribe.cgroup; SMqConsumerObj *pExistedConsumer = NULL; SMqConsumerObj *pConsumerNew = NULL; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6dab018236..83e8967c14 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -183,33 +183,33 @@ void tFreeStreamObj(SStreamObj *pStream) { } } -SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { - SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); - if (pVgEpNew == NULL) return NULL; - pVgEpNew->vgId = pVgEp->vgId; - pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); - pVgEpNew->epSet = pVgEp->epSet; - return pVgEpNew; -} +//SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { +// SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); +// if (pVgEpNew == NULL) return NULL; +// pVgEpNew->vgId = pVgEp->vgId; +//// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); +// pVgEpNew->epSet = pVgEp->epSet; +// return pVgEpNew; +//} -void tDeleteSMqVgEp(SMqVgEp *pVgEp) { - if (pVgEp) { - taosMemoryFreeClear(pVgEp->qmsg); - taosMemoryFree(pVgEp); - } -} +//void tDeleteSMqVgEp(SMqVgEp *pVgEp) { +// if (pVgEp) { +//// taosMemoryFreeClear(pVgEp->qmsg); +// taosMemoryFree(pVgEp); +// } +//} int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeString(buf, pVgEp->qmsg); +// tlen += taosEncodeString(buf, pVgEp->qmsg); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeString(buf, &pVgEp->qmsg); +// buf = taosDecodeString(buf, &pVgEp->qmsg); buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return (void *)buf; } @@ -382,13 +382,13 @@ SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); if (pConsumerEpNew == NULL) return NULL; pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; - pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp); + pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL); return pConsumerEpNew; } void tDeleteSMqConsumerEp(void *data) { SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; - taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); + taosArrayDestroy(pConsumerEp->vgs); } int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { @@ -463,12 +463,13 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerEp newEp = { .consumerId = pConsumerEp->consumerId, - .vgs = taosArrayDup(pConsumerEp->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp), + .vgs = taosArrayDup(pConsumerEp->vgs, NULL), }; taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp)); } - pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp); + pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, NULL); memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); + pSubNew->qmsg = taosStrdup(pSub->qmsg); return pSubNew; } @@ -478,10 +479,11 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { pIter = taosHashIterate(pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); + taosArrayDestroy(pConsumerEp->vgs); } taosHashCleanup(pSub->consumerHash); - taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp); + taosArrayDestroy(pSub->unassignedVgs); + taosMemoryFreeClear(pSub->qmsg); } int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 64082536da..9a611fe46a 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -570,25 +570,21 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); - if (pSubplan) { - int32_t msgLen; - - pSubplan->execNode.epSet = pVgEp->epSet; - pSubplan->execNode.nodeId = pVgEp->vgId; - - if (qSubPlanToString(pSubplan, &pVgEp->qmsg, &msgLen) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } - } else { - pVgEp->qmsg = taosStrdup(""); - } - sdbRelease(pSdb, pVgroup); } + if (pSubplan) { + int32_t msgLen; + + if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + } else { + pSub->qmsg = taosStrdup(""); + } + qDestroyQueryPlan(pPlan); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 74421afa33..42af4944a5 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -99,13 +99,23 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj return pSub; } -static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscribeObj *pSub, - const SMqRebOutputVg *pRebVg) { +static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; req.vgId = pRebVg->pVgEp->vgId; - req.qmsg = pRebVg->pVgEp->qmsg; + if(pPlan){ + pPlan->execNode.epSet = pRebVg->pVgEp->epSet; + pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; + int32_t msgLen; + if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + }else{ + req.qmsg = taosStrdup(""); + } req.subType = pSub->subType; req.withMeta = pSub->withMeta; req.suid = pSub->stbUid; @@ -115,6 +125,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri int32_t ret = 0; tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); if (ret < 0) { + taosMemoryFree(req.qmsg); return -1; } @@ -122,6 +133,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(req.qmsg); return -1; } @@ -135,17 +147,19 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { taosMemoryFreeClear(buf); tEncoderClear(&encoder); + taosMemoryFree(req.qmsg); return -1; } tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; + taosMemoryFree(req.qmsg); return 0; } -static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, - const SMqRebOutputVg *pRebVg) { +static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { // if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { // terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; // return -1; @@ -153,7 +167,7 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM void *buf; int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg) < 0) { + if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) { return -1; } @@ -483,14 +497,25 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { + struct SSubplan* pPlan = NULL; + if(strcmp(pOutput->pSub->qmsg, "") != 0){ + int32_t code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) { + nodesDestroyNode((SNode*)pPlan); return -1; } mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL); if (mndTrancCheckConflict(pMnode, pTrans) != 0) { mndTransDrop(pTrans); + nodesDestroyNode((SNode*)pPlan); return -1; } @@ -500,11 +525,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu int32_t vgNum = taosArrayGetSize(rebVgs); for (int32_t i = 0; i < vgNum; i++) { SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); - if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) { + if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg, pPlan) < 0) { mndTransDrop(pTrans); + nodesDestroyNode((SNode*)pPlan); return -1; } } + nodesDestroyNode((SNode*)pPlan); // 2. redo log: subscribe and vg assignment // subscribe From 48b9cac75c4a81050463616a13075b3cdd2b9c3b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 13 Jun 2023 11:44:13 +0800 Subject: [PATCH 110/122] fix: fix no data in db issue --- source/libs/parser/src/parTranslater.c | 20 +++++++++++++------- source/libs/parser/test/parInitialCTest.cpp | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 905b32a5f3..b6191dfae5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6932,20 +6932,26 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot; STranslateContext cxt = {0}; SInterval interval = {0}; - - if (NULL == pResRow || NULL == pResRow[0]) { - return TSDB_CODE_PAR_INTERNAL_ERROR; - } + int64_t lastTs = 0; int32_t code = initTranslateContext(pParseCxt, NULL, &cxt); if (TSDB_CODE_SUCCESS == code) { code = buildIntervalForCreateStream(pStmt, &interval); } if (TSDB_CODE_SUCCESS == code) { - if (interval.interval > 0) { - pStmt->pReq->lastTs = taosTimeTruncate(*(int64_t*)pResRow[0], &interval); + if (pResRow && pResRow[0]) { + lastTs = *(int64_t*)pResRow[0]; + } else if (interval.interval > 0) { + lastTs = convertTimePrecision(taosGetTimestampMs(), TSDB_TIME_PRECISION_MILLI, interval.precision); } else { - pStmt->pReq->lastTs = *(int64_t*)pResRow[0]; + lastTs = taosGetTimestampMs(); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (interval.interval > 0) { + pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + } else { + pStmt->pReq->lastTs = lastTs; } code = buildCmdMsg(&cxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, pStmt->pReq); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index a4e8bdd87a..f6dfa93ab2 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -885,12 +885,12 @@ TEST_F(ParserInitialCTest, createStream) { setCreateStreamReq( "s1", "test", - "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 ignore " + "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 0 ignore " "update 1 into st3 as select count(*) from t1 interval(10s)", "st3", 1); setStreamOptions(STREAM_CREATE_STABLE_TRUE, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, - 10 * MILLISECOND_PER_SECOND, 0, 1, 1); - run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 IGNORE " + 10 * MILLISECOND_PER_SECOND, 0, 0, 1); + run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 0 IGNORE " "UPDATE 1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); From b8b3036905e893ec0424445c88eb44f8939be7e5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 13 Jun 2023 13:53:40 +0800 Subject: [PATCH 111/122] fix: fix case issue --- tests/system-test/2-query/odbc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index 5241406b65..9f51bc58da 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,8 +22,8 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - # enterprise version: 288, community version: 280 - tdSql.checkData(0, 0, 288) + # enterprise version: 289, community version: 281 + tdSql.checkData(0, 0, 289) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) From 28e6c4502122cd5c79d5b111dbfde1d0ee69b424 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 13 Jun 2023 14:41:42 +0800 Subject: [PATCH 112/122] enhance: version not compatible log --- source/client/src/clientMsgHandler.c | 1 + source/dnode/mnode/impl/src/mndProfile.c | 1 + 2 files changed, 2 insertions(+) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 6d53f2b4c5..d6fdb29b59 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -77,6 +77,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { } if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) { + tscError("version not compatible. client version: %s, server version: %s", version, connectRsp.sVer); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); goto End; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index a1d815189c..aeec7eac00 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -227,6 +227,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) { + mGError("version not compatible. client version: %s, server version: %s", connReq.sVer, version); terrno = code; goto _OVER; } From 13ddd37529157ef54ebf631928c8e23d94ba2ac9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 13 Jun 2023 14:51:41 +0800 Subject: [PATCH 113/122] fix: fix memory sanitizer error --- source/libs/executor/src/tsort.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 783597df67..3033441aad 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -101,7 +101,11 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); + if (pSource->pageIdList) { + taosArrayDestroy(pSource->pageIdList); + } taosMemoryFreeClear(pSource); + cmpParam->pSources[i] = NULL; } cmpParam->numOfSources = 0; @@ -123,9 +127,11 @@ void tsortClearOrderdSource(SArray* pOrderedSource, int64_t *fetchUs, int64_t *f // release pageIdList if ((*pSource)->pageIdList) { taosArrayDestroy((*pSource)->pageIdList); + (*pSource)->pageIdList = NULL; } if ((*pSource)->param && !(*pSource)->onlyRef) { taosMemoryFree((*pSource)->param); + (*pSource)->param = NULL; } if (!(*pSource)->onlyRef && (*pSource)->src.pBlock) { From 8a81278024a05a664ec07c0e09c4ee56f7cbe8e9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Jun 2023 15:14:11 +0800 Subject: [PATCH 114/122] fix:load pHandle if necessary wher tq restore --- include/libs/wal/wal.h | 2 +- include/util/talgo.h | 2 +- include/util/tarray.h | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndDef.c | 38 +++-- source/dnode/vnode/src/inc/tq.h | 3 + source/dnode/vnode/src/tq/tq.c | 101 ++----------- source/dnode/vnode/src/tq/tqMeta.c | 217 +++++++++++++++++---------- source/libs/wal/src/walRef.c | 19 +-- source/util/src/tarray.c | 4 +- 10 files changed, 186 insertions(+), 204 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 1aa08ff802..47230bc95c 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -214,7 +214,7 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); void walRefFirstVer(SWal *, SWalRef *); void walRefLastVer(SWal *, SWalRef *); -SWalRef *walRefCommittedVer(SWal *); +void walRefCommitVer(SWal *, SWalRef *); SWalRef *walOpenRef(SWal *); void walCloseRef(SWal *pWal, int64_t refId); diff --git a/include/util/talgo.h b/include/util/talgo.h index f9d51c4b5b..7c92c0fe87 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -31,7 +31,7 @@ typedef void *(*__array_item_dup_fn_t)(void *); typedef void (*FDelete)(void *); typedef int32_t (*FEncode)(void **buf, const void *dst); -typedef void *(*FDecode)(const void *buf, void *dst); +typedef void *(*FDecode)(const void *buf, void *dst, int8_t sver); #define TD_EQ 0x1 #define TD_GT 0x2 diff --git a/include/util/tarray.h b/include/util/tarray.h index 4bf24b46b9..a93c695370 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -244,7 +244,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); -void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 3a3050839b..4a90f3939f 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -572,7 +572,7 @@ typedef struct { //SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); //void tDeleteSMqVgEp(SMqVgEp* pVgEp); int32_t tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp); -void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp); +void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp, int8_t sver); typedef struct { int64_t consumerId; // -1 for unassigned diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b6cc962d16..15f0cc9b71 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -207,9 +207,13 @@ int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { return tlen; } -void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { +void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); -// buf = taosDecodeString(buf, &pVgEp->qmsg); + if(sver == 1){ + uint64_t size = 0; + buf = taosDecodeVariantU64(buf, &size); + buf = POINTER_SHIFT(buf, size); + } buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return (void *)buf; } @@ -390,18 +394,18 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s return (void *)buf; } -SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { - SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); - if (pConsumerEpNew == NULL) return NULL; - pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; - pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL); - return pConsumerEpNew; -} - -void tDeleteSMqConsumerEp(void *data) { - SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; - taosArrayDestroy(pConsumerEp->vgs); -} +//SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { +// SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); +// if (pConsumerEpNew == NULL) return NULL; +// pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; +// pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL); +// return pConsumerEpNew; +//} +// +//void tDeleteSMqConsumerEp(void *data) { +// SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; +// taosArrayDestroy(pConsumerEp->vgs); +//} int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { int32_t tlen = 0; @@ -436,7 +440,7 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) { buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); + buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver); if (sver > 1){ int32_t szVgs = 0; buf = taosDecodeFixedI32(buf, &szVgs); @@ -580,6 +584,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { // do nothing } } + tlen += taosEncodeString(buf, pSub->qmsg); return tlen; } @@ -602,7 +607,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp)); } - buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); + buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver); buf = taosDecodeStringTo(buf, pSub->dbName); if (sver > 1){ @@ -626,6 +631,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { } } } + buf = taosDecodeString(buf, &pSub->qmsg); } return (void *)buf; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 4ba8d6d69f..b35dc71ed9 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -139,6 +139,7 @@ static STqMgmt tqMgmt = {0}; int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); +void tqDestroyTqHandle(void* data); // tqRead int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); @@ -161,6 +162,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq); int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen); int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); +int32_t tqMetaGetHandle(STQ* pTq, const char* key); +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); STqOffsetStore* tqOffsetOpen(STQ* pTq); void tqOffsetClose(STqOffsetStore*); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index edefd3ba90..f4d3407aa2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -62,7 +62,7 @@ void tqCleanUp() { } } -static void destroyTqHandle(void* data) { +void tqDestroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); @@ -102,7 +102,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->walLogLastVer = pVnode->pWal->vers.lastVer; pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); - taosHashSetFreeFp(pTq->pHandle, destroyTqHandle); + taosHashSetFreeFp(pTq->pHandle, tqDestroyTqHandle); taosInitRWLatch(&pTq->lock); pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); @@ -661,13 +661,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg return -1; } - SVnode* pVnode = pTq->pVnode; - int32_t vgId = TD_VID(pVnode); - - tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, + tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + STqHandle* pHandle = NULL; + while(1){ + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle || tqMetaGetHandle(pTq, req.subKey) < 0){ + break; + } + } + if (pHandle == NULL) { if (req.oldConsumerId != -1) { tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, @@ -678,86 +682,13 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqError("vgId:%d, tq invalid re-balance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId); goto end; } - - STqHandle tqHandle = {0}; - pHandle = &tqHandle; - - memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); - pHandle->consumerId = req.newConsumerId; - pHandle->epoch = -1; - - pHandle->execHandle.subType = req.subType; - pHandle->fetchMeta = req.withMeta; - - // TODO version should be assigned and refed during preprocess - SWalRef* pRef = walRefCommittedVer(pVnode->pWal); - if (pRef == NULL) { - ret = -1; + STqHandle handle = {0}; + ret = tqCreateHandle(pTq, &req, &handle); + if(ret < 0){ + tqDestroyTqHandle(&handle); goto end; } - - int64_t ver = pRef->refVer; - pHandle->pRef = pRef; - - SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; - initStorageAPI(&handle.api); - - pHandle->snapshotVer = ver; - - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - pHandle->execHandle.execCol.qmsg = taosStrdup(req.qmsg); - - pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, - &pHandle->execHandle.numOfCols, req.newConsumerId); - void* scanner = NULL; - qExtractStreamScanner(pHandle->execHandle.task, &scanner); - pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); - } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { - pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); - - pHandle->execHandle.execDb.pFilterOutTbUid = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - buildSnapContext(handle.vnode, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, - (SSnapContext**)(&handle.sContext)); - - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); - } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.execTb.suid = req.suid; - pHandle->execHandle.execTb.qmsg = taosStrdup(req.qmsg); - - if (strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { - if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { - tqError("nodesStringToNode error in sub stable, since %s, vgId:%d, subkey:%s consumer:0x%" PRIx64, terrstr(), - pVnode->config.vgId, req.subKey, pHandle->consumerId); - return -1; - } - } - - buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, - (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); - - SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList, pHandle->execHandle.task); - if (ret != TDB_CODE_SUCCESS) { - tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, - pHandle->consumerId); - taosArrayDestroy(tbUidList); - goto end; - } - tqDebug("tq try to get ctb for stb subscribe, vgId:%d, subkey:%s consumer:0x%" PRIx64 " suid:%" PRId64, - pVnode->config.vgId, req.subKey, pHandle->consumerId, req.suid); - pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList, NULL); - taosArrayDestroy(tbUidList); - } - - taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); - tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, pHandle->consumerId); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; + ret = tqMetaSaveHandle(pTq, req.subKey, &handle); } else { taosWLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index ba6d7cb501..3b0e6749c2 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -88,9 +88,9 @@ int32_t tqMetaOpen(STQ* pTq) { return -1; } - if (tqMetaRestoreHandle(pTq) < 0) { - return -1; - } +// if (tqMetaRestoreHandle(pTq) < 0) { +// return -1; +// } if (tqMetaRestoreCheckInfo(pTq) < 0) { return -1; @@ -274,6 +274,120 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { return 0; } +static int buildHandle(STQ* pTq, STqHandle* handle){ + SVnode* pVnode = pTq->pVnode; + int32_t vgId = TD_VID(pVnode); + + handle->pRef = walOpenRef(pVnode->pWal); + if (handle->pRef == NULL) { + return -1; + } + walSetRefVer(handle->pRef, handle->snapshotVer); + + SReadHandle reader = { + .vnode = pVnode, + .initTableReader = true, + .initTqReader = true, + .version = handle->snapshotVer, + }; + + initStorageAPI(&reader.api); + + if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + handle->execHandle.task = + qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); + if (handle->execHandle.task == NULL) { + tqError("cannot create exec task for %s", handle->subKey); + return -1; + } + void* scanner = NULL; + qExtractStreamScanner(handle->execHandle.task, &scanner); + if (scanner == NULL) { + tqError("cannot extract stream scanner for %s", handle->subKey); + return -1; + } + handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); + if (handle->execHandle.pTqReader == NULL) { + tqError("cannot extract exec reader for %s", handle->subKey); + return -1; + } + } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) { + handle->pWalReader = walOpenReader(pVnode->pWal, NULL); + handle->execHandle.pTqReader = tqReaderOpen(pVnode); + + buildSnapContext(reader.vnode, reader.version, 0, handle->execHandle.subType, handle->fetchMeta, + (SSnapContext**)(&reader.sContext)); + handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); + } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + handle->pWalReader = walOpenReader(pVnode->pWal, NULL); + + if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { + if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; + } + } + buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType, + handle->fetchMeta, (SSnapContext**)(&reader.sContext)); + handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); + + SArray* tbUidList = NULL; + int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList, handle->execHandle.task); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId); + taosArrayDestroy(tbUidList); + return -1; + } + tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid); + handle->execHandle.pTqReader = tqReaderOpen(pVnode); + tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL); + taosArrayDestroy(tbUidList); + } + return 0; +} + +static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ + int32_t vgId = TD_VID(pTq->pVnode); + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + tDecodeSTqHandle(&decoder, handle); + tDecoderClear(&decoder); + + if(buildHandle(pTq, handle) < 0){ + return -1; + } + tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle->subKey, handle->consumerId, vgId); + return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); +} + +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ + int32_t vgId = TD_VID(pTq->pVnode); + + memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); + handle->consumerId = req->newConsumerId; + handle->epoch = -1; + + handle->execHandle.subType = req->subType; + handle->fetchMeta = req->withMeta; + if(req->subType == TOPIC_SUB_TYPE__COLUMN){ + handle->execHandle.execCol.qmsg = taosStrdup(req->qmsg); + }else if(req->subType == TOPIC_SUB_TYPE__DB){ + handle->execHandle.execDb.pFilterOutTbUid = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + }else if(req->subType == TOPIC_SUB_TYPE__TABLE){ + handle->execHandle.execTb.suid = req->suid; + handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg); + } + + handle->snapshotVer = walGetLastVer(pTq->pVnode->pWal); + + if(buildHandle(pTq, handle) < 0){ + return -1; + } + tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle->subKey, handle->consumerId, vgId); + return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); +} + int32_t tqMetaRestoreHandle(STQ* pTq) { int code = 0; TBC* pCur = NULL; @@ -281,97 +395,40 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { return -1; } - int32_t vgId = TD_VID(pTq->pVnode); void* pKey = NULL; int kLen = 0; void* pVal = NULL; int vLen = 0; - SDecoder decoder; tdbTbcMoveToFirst(pCur); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { STqHandle handle = {0}; - tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeSTqHandle(&decoder, &handle); - tDecoderClear(&decoder); - - handle.pRef = walOpenRef(pTq->pVnode->pWal); - if (handle.pRef == NULL) { - code = -1; - goto end; + code = restoreHandle(pTq, pVal, vLen, &handle); + if (code < 0){ + tqDestroyTqHandle(&handle); + break; } - walSetRefVer(handle.pRef, handle.snapshotVer); - - SReadHandle reader = { - .vnode = pTq->pVnode, - .initTableReader = true, - .initTqReader = true, - .version = handle.snapshotVer - }; - - initStorageAPI(&reader.api); - - if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - handle.execHandle.task = - qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); - if (handle.execHandle.task == NULL) { - tqError("cannot create exec task for %s", handle.subKey); - code = -1; - goto end; - } - void* scanner = NULL; - qExtractStreamScanner(handle.execHandle.task, &scanner); - if (scanner == NULL) { - tqError("cannot extract stream scanner for %s", handle.subKey); - code = -1; - goto end; - } - handle.execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); - if (handle.execHandle.pTqReader == NULL) { - tqError("cannot extract exec reader for %s", handle.subKey); - code = -1; - goto end; - } - } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { - handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - - buildSnapContext(reader.vnode, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, - (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); - } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - - if(handle.execHandle.execTb.qmsg != NULL && strcmp(handle.execHandle.execTb.qmsg, "") != 0) { - if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { - tqError("nodesStringToNode error in sub stable, since %s", terrstr()); - return -1; - } - } - buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, - handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); - - SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList, handle.execHandle.task); - if(ret != TDB_CODE_SUCCESS) { - tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); - taosArrayDestroy(tbUidList); - goto end; - } - tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); - handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList, NULL); - taosArrayDestroy(tbUidList); - } - tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); - taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); } -end: tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return code; } + +int32_t tqMetaGetHandle(STQ* pTq, const char* key) { + void* pVal = NULL; + int vLen = 0; + + if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &pVal, &vLen) < 0) { + return -1; + } + STqHandle handle = {0}; + int code = restoreHandle(pTq, pVal, vLen, &handle); + if (code < 0){ + tqDestroyTqHandle(&handle); + } + tdbFree(pVal); + return code; +} diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index eb36389f1d..2f1bcfee83 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -81,26 +81,11 @@ void walRefLastVer(SWal *pWal, SWalRef *pRef) { wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); } -SWalRef *walRefCommittedVer(SWal *pWal) { - SWalRef *pRef = walOpenRef(pWal); - if (pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } +void walRefCommitVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); - int64_t ver = walGetCommittedVer(pWal); - - wDebug("vgId:%d, wal ref version %" PRId64 " for committed", pWal->cfg.vgId, ver); - pRef->refVer = ver; - // bsearch in fileSet - SWalFileInfo tmpInfo; - tmpInfo.firstVer = ver; - SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); - // pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); - return pRef; + wDebug("vgId:%d, wal ref version %" PRId64 " for committed", pWal->cfg.vgId, ver); } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 6c7c5ddb0d..8906391a9a 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -476,13 +476,13 @@ int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) { return tlen; } -void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz) { +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); *pArray = taosArrayInit(sz, sizeof(void*)); for (int32_t i = 0; i < sz; i++) { void* data = taosMemoryCalloc(1, dataSz); - buf = decode(buf, data); + buf = decode(buf, data, sver); taosArrayPush(*pArray, &data); } return (void*)buf; From 827e84b1a5523f40c5f7f092aa34cf58362a85e9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Jun 2023 17:10:23 +0800 Subject: [PATCH 115/122] fix:memory leak --- source/dnode/mnode/impl/inc/mndDef.h | 4 ++-- source/dnode/mnode/impl/src/mndDef.c | 36 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index af66f8d149..62beee0303 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -570,8 +570,8 @@ typedef struct { SEpSet epSet; } SMqVgEp; -//SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); -//void tDeleteSMqVgEp(SMqVgEp* pVgEp); +SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); +void tDeleteSMqVgEp(SMqVgEp* pVgEp); int32_t tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp); void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp, int8_t sver); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 15f0cc9b71..f0fa40cacf 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -183,21 +183,21 @@ void tFreeStreamObj(SStreamObj *pStream) { } } -//SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { -// SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); -// if (pVgEpNew == NULL) return NULL; -// pVgEpNew->vgId = pVgEp->vgId; -//// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); -// pVgEpNew->epSet = pVgEp->epSet; -// return pVgEpNew; -//} +SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { + SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); + if (pVgEpNew == NULL) return NULL; + pVgEpNew->vgId = pVgEp->vgId; +// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); + pVgEpNew->epSet = pVgEp->epSet; + return pVgEpNew; +} -//void tDeleteSMqVgEp(SMqVgEp *pVgEp) { -// if (pVgEp) { -//// taosMemoryFreeClear(pVgEp->qmsg); -// taosMemoryFree(pVgEp); -// } -//} +void tDeleteSMqVgEp(SMqVgEp *pVgEp) { + if (pVgEp) { +// taosMemoryFreeClear(pVgEp->qmsg); + taosMemoryFree(pVgEp); + } +} int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { int32_t tlen = 0; @@ -517,11 +517,11 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerEp newEp = { .consumerId = pConsumerEp->consumerId, - .vgs = taosArrayDup(pConsumerEp->vgs, NULL), + .vgs = taosArrayDup(pConsumerEp->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp), }; taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp)); } - pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, NULL); + pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp); pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL); memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); pSubNew->qmsg = taosStrdup(pSub->qmsg); @@ -534,11 +534,11 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { pIter = taosHashIterate(pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - taosArrayDestroy(pConsumerEp->vgs); + taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); taosArrayDestroy(pConsumerEp->offsetRows); } taosHashCleanup(pSub->consumerHash); - taosArrayDestroy(pSub->unassignedVgs); + taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp); taosMemoryFreeClear(pSub->qmsg); taosArrayDestroy(pSub->offsetRows); } From 7e6c7e767ade93921ddfaca04dd52c34c080cdef Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Jun 2023 19:10:45 +0800 Subject: [PATCH 116/122] fix:Vnode is closed or removed if drop database --- source/client/src/clientSml.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index bea237d09e..efa955ec84 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1756,9 +1756,8 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, request->code = code; info->cost.endTime = taosGetTimestampUs(); info->cost.code = code; - if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING || - code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT || - code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + if (NEED_CLIENT_HANDLE_ERROR(code) || code == TSDB_CODE_SDB_OBJ_CREATING || + code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) { if (cnt++ >= 10) { uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); break; From c28d7d9bf863e6dbbc6e35b96182a2c9bcdf9400 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 14 Jun 2023 12:01:54 +0800 Subject: [PATCH 117/122] fix:open info log in tmq client & modify parametes in show consumers --- include/common/tmsg.h | 3 - source/client/src/clientTmq.c | 109 +++++++++--------- source/common/src/systable.c | 6 +- source/dnode/mnode/impl/inc/mndDef.h | 1 - source/dnode/mnode/impl/src/mndConsumer.c | 24 ++-- source/dnode/mnode/impl/src/mndDef.c | 2 - tests/system-test/2-query/odbc.py | 4 +- .../system-test/7-tmq/checkOffsetRowParams.py | 4 + 8 files changed, 69 insertions(+), 84 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index aa0a243e68..f42defc788 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2035,7 +2035,6 @@ typedef struct { SArray* topicNames; // SArray int8_t withTbName; - int8_t useSnapshot; int8_t autoCommit; int32_t autoCommitInterval; int8_t resetOffsetCfg; @@ -2055,7 +2054,6 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc } tlen += taosEncodeFixedI8(buf, pReq->withTbName); - tlen += taosEncodeFixedI8(buf, pReq->useSnapshot); tlen += taosEncodeFixedI8(buf, pReq->autoCommit); tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval); tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg); @@ -2079,7 +2077,6 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq } buf = taosDecodeFixedI8(buf, &pReq->withTbName); - buf = taosDecodeFixedI8(buf, &pReq->useSnapshot); buf = taosDecodeFixedI8(buf, &pReq->autoCommit); buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval); buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 26887e2ade..e7927cd0ae 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -358,7 +358,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } - if (strcasecmp(key, "enable.heartbeat.background") == 0) { +// if (strcasecmp(key, "enable.heartbeat.background") == 0) { // if (strcasecmp(value, "true") == 0) { // conf->hbBgEnable = true; // return TMQ_CONF_OK; @@ -366,10 +366,10 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value // conf->hbBgEnable = false; // return TMQ_CONF_OK; // } else { - tscError("the default value of enable.heartbeat.background is true, can not be seted"); - return TMQ_CONF_INVALID; +// tscError("the default value of enable.heartbeat.background is true, can not be seted"); +// return TMQ_CONF_INVALID; // } - } +// } if (strcasecmp(key, "td.connect.ip") == 0) { conf->ip = taosStrdup(value); @@ -423,30 +423,30 @@ char** tmq_list_to_c_array(const tmq_list_t* list) { return container->pData; } -static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index, - int32_t* numOfVgroups) { - int32_t numOfTopics = taosArrayGetSize(pTopicList); - *index = -1; - *numOfVgroups = 0; - - for (int32_t i = 0; i < numOfTopics; ++i) { - SMqClientTopic* pTopic = taosArrayGet(pTopicList, i); - if (strcmp(pTopic->topicName, pName) != 0) { - continue; - } - - *numOfVgroups = taosArrayGetSize(pTopic->vgs); - for (int32_t j = 0; j < (*numOfVgroups); ++j) { - SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (pClientVg->vgId == vgId) { - *index = j; - return pClientVg; - } - } - } - - return NULL; -} +//static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index, +// int32_t* numOfVgroups) { +// int32_t numOfTopics = taosArrayGetSize(pTopicList); +// *index = -1; +// *numOfVgroups = 0; +// +// for (int32_t i = 0; i < numOfTopics; ++i) { +// SMqClientTopic* pTopic = taosArrayGet(pTopicList, i); +// if (strcmp(pTopic->topicName, pName) != 0) { +// continue; +// } +// +// *numOfVgroups = taosArrayGetSize(pTopic->vgs); +// for (int32_t j = 0; j < (*numOfVgroups); ++j) { +// SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); +// if (pClientVg->vgId == vgId) { +// *index = j; +// return pClientVg; +// } +// } +// } +// +// return NULL; +//} // Two problems do not need to be addressed here // 1. update to of epset. the response of poll request will automatically handle this problem @@ -573,7 +573,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN char commitBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, + tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, tmq->consumerId, pOffset->offset.subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, totalVgroups, pMsgSendInfo->requestId); @@ -811,7 +811,9 @@ void tmqSendHbReq(void* param, void* tmrId) { offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; offRows->offset = pVg->offsetInfo.committedOffset; - tscDebug("report offset: %d", offRows->offset.type); + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); + tscInfo("report offset: vgId:%d, offset:%s, rows:%"PRId64, offRows->vgId, buf, offRows->rows); } } tmq->needReportOffsetRows = false; @@ -862,7 +864,7 @@ OVER: static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { if (code != 0) { - tscDebug("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); + tscError("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); } } @@ -1161,7 +1163,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = 0; - tscDebug("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); + tscInfo("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); @@ -1174,7 +1176,6 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } req.withTbName = tmq->withTbName; - req.useSnapshot = tmq->useSnapshot; req.autoCommit = tmq->autoCommit; req.autoCommitInterval = tmq->autoCommitInterval; req.resetOffsetCfg = tmq->resetOffsetCfg; @@ -1190,7 +1191,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } tNameExtractFullName(&name, topicFName); - tscDebug("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); + tscInfo("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); taosArrayPush(req.topicNames, &topicFName); } @@ -1251,7 +1252,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - tscDebug("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + tscInfo("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); taosMsleep(500); } @@ -1478,7 +1479,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); - tscDebug("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); + tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); for (int32_t j = 0; j < vgNumGet; j++) { @@ -1531,7 +1532,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", + tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); if (epoch <= tmq->epoch) { return false; @@ -1554,14 +1555,14 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); if (pTopicCur->vgs) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); + tscInfo("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); char buf[TSDB_OFFSET_LEN]; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.currentOffset); - tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, + tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); SVgroupSaveInfo info = {.offset = pVgCur->offsetInfo.currentOffset, .numOfRows = pVgCur->numOfRows}; @@ -1591,7 +1592,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) atomic_store_8(&tmq->status, flag); atomic_store_32(&tmq->epoch, epoch); - tscDebug("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); return set; } @@ -1627,7 +1628,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); if (head->epoch <= epoch) { - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", + tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", tmq->consumerId, head->epoch, epoch); if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) { @@ -1639,7 +1640,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { } } else { - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, + tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, head->epoch, epoch); } @@ -2067,12 +2068,12 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj; int64_t startTime = taosGetTimestampMs(); - tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, + tscInfo("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, timeout); // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { - tscDebug("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); taosMsleep(500); // sleep for a while return NULL; } @@ -2084,7 +2085,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - tscDebug("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt); + tscInfo("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt); taosMsleep(500); } } @@ -2093,7 +2094,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) { - tscDebug("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); + tscError("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); } rspObj = tmqHandleAllRsp(tmq, timeout, false); @@ -2101,7 +2102,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - tscDebug("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); return NULL; } @@ -2109,7 +2110,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout) { - tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, + tscInfo("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } @@ -2142,7 +2143,7 @@ static void displayConsumeStatistics(const tmq_t* pTmq) { } int32_t tmq_consumer_close(tmq_t* tmq) { - tscDebug("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); + tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); displayConsumeStatistics(tmq); if (tmq->status == TMQ_CONSUMER_STATUS__READY) { @@ -2169,7 +2170,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { tmq_list_destroy(lst); } else { - tscWarn("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } taosRemoveRef(tmqMgmt.rsetId, tmq->refId); @@ -2432,7 +2433,7 @@ void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) { sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + tscInfo("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); int64_t transporterId = 0; asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); @@ -2656,7 +2657,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char offsetFormatBuf[TSDB_OFFSET_LEN]; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); - tscDebug("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, + tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); } @@ -2693,7 +2694,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char offsetBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffsetInfo->currentOffset); - tscDebug("vgId:%d offset is update to:%s", p->vgId, offsetBuf); + tscInfo("vgId:%d offset is update to:%s", p->vgId, offsetBuf); pOffsetInfo->walVerBegin = p->begin; pOffsetInfo->walVerEnd = p->end; @@ -2772,7 +2773,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId}; tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic)); - tscDebug("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); + tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 722092a043..2c8f484a32 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -361,11 +361,7 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "msg.with.table.name", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, - {.name = "experimental.snapshot.enable", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, - {.name = "enable.auto.commit", .bytes = 1, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = false}, - {.name = "auto.commit.interval.ms", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, - {.name = "auto.offset.reset", .bytes = TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "parameters", .bytes = 64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, }; static const SSysDbTableSchema offsetSchema[] = { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index f74a593b00..2dcba291cd 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -552,7 +552,6 @@ typedef struct { int64_t rebalanceTime; int8_t withTbName; - int8_t useSnapshot; int8_t autoCommit; int32_t autoCommitInterval; int32_t resetOffsetCfg; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index b5254dbb88..7730ab054d 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -697,7 +697,6 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId)); pConsumerNew->withTbName = subscribe.withTbName; - pConsumerNew->useSnapshot = subscribe.useSnapshot; pConsumerNew->autoCommit = subscribe.autoCommit; pConsumerNew->autoCommitInterval = subscribe.autoCommitInterval; pConsumerNew->resetOffsetCfg = subscribe.resetOffsetCfg; @@ -1186,25 +1185,16 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->withTbName, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->useSnapshot, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->autoCommit, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->autoCommitInterval, false); - - char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg}; - tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &pVal); - varDataSetLen(buf, strlen(varDataVal(buf))); + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal); + + char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%d,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); + varDataSetLen(parasStr, strlen(varDataVal(parasStr))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)buf, false); + colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 7f79408e8c..2e8a937f07 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -322,7 +322,6 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { } tlen += taosEncodeFixedI8(buf, pConsumer->withTbName); - tlen += taosEncodeFixedI8(buf, pConsumer->useSnapshot); tlen += taosEncodeFixedI8(buf, pConsumer->autoCommit); tlen += taosEncodeFixedI32(buf, pConsumer->autoCommitInterval); tlen += taosEncodeFixedI32(buf, pConsumer->resetOffsetCfg); @@ -382,7 +381,6 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s if(sver > 1){ buf = taosDecodeFixedI8(buf, &pConsumer->withTbName); - buf = taosDecodeFixedI8(buf, &pConsumer->useSnapshot); buf = taosDecodeFixedI8(buf, &pConsumer->autoCommit); buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval); buf = taosDecodeFixedI32(buf, &pConsumer->resetOffsetCfg); diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index b6d2ab2a3f..ea897260d1 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,8 +22,8 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - # enterprise version: 295, community version: 285 - tdSql.checkData(0, 0, 295) + # enterprise version: 291, community version: 281 + tdSql.checkData(0, 0, 291) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) diff --git a/tests/system-test/7-tmq/checkOffsetRowParams.py b/tests/system-test/7-tmq/checkOffsetRowParams.py index 17c80c68bf..8a24148064 100644 --- a/tests/system-test/7-tmq/checkOffsetRowParams.py +++ b/tests/system-test/7-tmq/checkOffsetRowParams.py @@ -243,6 +243,10 @@ class TDTestCase: tdSql.checkData(0, 5, 0) break + tdSql.query("show consumers") + tdSql.checkRows(1) + tdSql.checkData(0, 8, "tbname:1,commit:1,interval:2000,reset:earliest") + time.sleep(2) tdLog.info("start insert data") self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) From 05393d90539282eb3df95c5b97deed2f9caabed2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 14 Jun 2023 13:05:47 +0800 Subject: [PATCH 118/122] fix: make vmProcessCreateVnodeReq idempotent --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 814a155cfb..dd880a87c8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -265,6 +265,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); +#if 0 if (pMgmt->pTfs) { if (tfsDirExistAt(pMgmt->pTfs, path, (SDiskID){0})) { terrno = TSDB_CODE_VND_DIR_ALREADY_EXIST; @@ -278,8 +279,9 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } } +#endif -if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { + if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { tFreeSCreateVnodeReq(&req); dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); code = terrno; From 0fac3f7d845823c0718275a12fef53abe260d0ca Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 14 Jun 2023 19:27:47 +0800 Subject: [PATCH 119/122] fix: add compact split resume stream pause stream to tab key --- tools/shell/src/shellAuto.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 19a888fe82..943842947a 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -91,9 +91,14 @@ SWords shellCommands[] = { {"create stream into as select", 0, 0, NULL}, // 26 append sub sql {"create topic as select", 0, 0, NULL}, // 27 append sub sql {"create function as outputtype language ", 0, 0, NULL}, + {"create or replace as outputtype language ", 0, 0, NULL}, {"create aggregate function as outputtype bufsize language ", 0, 0, NULL}, + {"create or replace aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"compact database ", 0, 0, NULL}, +#endif {"describe ", 0, 0, NULL}, {"delete from where ", 0, 0, NULL}, {"drop database ", 0, 0, NULL}, @@ -117,7 +122,11 @@ SWords shellCommands[] = { {"kill connection ;", 0, 0, NULL}, {"kill query ", 0, 0, NULL}, {"kill transaction ", 0, 0, NULL}, +#ifdef TD_ENTERPRISE {"merge vgroup ", 0, 0, NULL}, +#endif + {"pause stream ;", 0, 0, NULL}, + {"resume stream ;", 0, 0, NULL}, {"reset query cache;", 0, 0, NULL}, {"restore dnode ;", 0, 0, NULL}, {"restore vnode on dnode ;", 0, 0, NULL}, @@ -173,7 +182,9 @@ SWords shellCommands[] = { {"show vgroups;", 0, 0, NULL}, {"show consumers;", 0, 0, NULL}, {"show grants;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE {"split vgroup ", 0, 0, NULL}, +#endif {"insert into values(", 0, 0, NULL}, {"insert into using tags(", 0, 0, NULL}, {"insert into using values(", 0, 0, NULL}, @@ -432,8 +443,6 @@ void showHelp() { kill connection ; \n\ kill query ; \n\ kill transaction ;\n\ - ----- M ----- \n\ - merge vgroup ...\n\ ----- R ----- \n\ reset query cache;\n\ restore dnode ;\n\ @@ -489,14 +498,20 @@ void showHelp() { show vgroups;\n\ show consumers;\n\ show grants;\n\ - split vgroup ...\n\ ----- T ----- \n\ trim database ;\n\ ----- U ----- \n\ use ;"); - printf("\n\n"); +#ifdef TD_ENTERPRISE + printf( + "\n\ + ----- special commands on enterpise version ----- \n\ + compact database ; \n\ + split vgroup ;\n"); +#endif + printf("\n\n"); // define in getDuration() function printf( "\ From 8018cbf63976132318e7d5555333d2962bb7b6b5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 14 Jun 2023 19:35:55 +0800 Subject: [PATCH 120/122] fix: add blank line --- tools/shell/src/shellAuto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 943842947a..27f3ae7a82 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -505,10 +505,10 @@ void showHelp() { #ifdef TD_ENTERPRISE printf( - "\n\ + "\n\n\ ----- special commands on enterpise version ----- \n\ compact database ; \n\ - split vgroup ;\n"); + split vgroup ;"); #endif printf("\n\n"); From d7f03678b3ce537956a246796c5c5035a9058d09 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 14 Jun 2023 19:54:32 +0800 Subject: [PATCH 121/122] fix: pause and resume stream add to show help --- tools/shell/src/shellAuto.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 27f3ae7a82..41cdb0f928 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -443,7 +443,10 @@ void showHelp() { kill connection ; \n\ kill query ; \n\ kill transaction ;\n\ + ----- P ----- \n\ + pause stream ;\n\ ----- R ----- \n\ + resume stream ;\n\ reset query cache;\n\ restore dnode ;\n\ restore vnode on dnode ;\n\ From b86fc3a697cc73ba4d0aeb59118f27106ab2804a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 14 Jun 2023 21:38:25 +0800 Subject: [PATCH 122/122] fix: vnodeRenameVgroupId correctly --- source/dnode/vnode/src/vnd/vnodeOpen.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b5e7c6875b..0655a46388 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -129,6 +129,12 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p return 0; } +static int32_t vnodeVgroupIdLen(int32_t vgId) { + char tmp[TSDB_FILENAME_LEN]; + sprintf(tmp, "%d", vgId); + return strlen(tmp); +} + int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) { int32_t ret = tfsRename(pTfs, srcPath, dstPath); if (ret != 0) return ret; @@ -154,8 +160,7 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + 6); if (tsdbFileVgId == srcVgId) { - char *tsdbFileSurfixPos = strstr(tsdbFilePrefixPos, "f"); - if (tsdbFileSurfixPos == NULL) continue; + char *tsdbFileSurfixPos = tsdbFilePrefixPos + 6 + vnodeVgroupIdLen(srcVgId); tsdbFilePrefixPos[6] = 0; snprintf(newRname, TSDB_FILENAME_LEN, "%s%d%s", oldRname, dstVgId, tsdbFileSurfixPos);