From d9401fd75db59bb21cc7d4ca03a0b1e70585af03 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 23 Jan 2025 09:37:07 +0000 Subject: [PATCH 01/71] feat/TS-5805-add-sql-command --- include/common/tmsg.h | 10 ++++++ include/common/tmsgdef.h | 1 + include/libs/nodes/cmdnodes.h | 4 +++ source/common/src/msg/tmsg.c | 40 +++++++++++++++++++++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mnode/impl/src/mndVgroup.c | 26 ++++++++++++++ source/libs/nodes/src/nodesCodeFuncs.c | 6 ++++ source/libs/nodes/src/nodesUtilFuncs.c | 4 +++ source/libs/parser/inc/parAst.h | 1 + source/libs/parser/inc/sql.y | 3 ++ source/libs/parser/src/parAstCreater.c | 12 ++++++- source/libs/parser/src/parTranslater.c | 14 ++++++++ 12 files changed, 121 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 82eaa2359e..670dc9453f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -359,6 +359,7 @@ typedef enum ENodeType { QUERY_NODE_CREATE_ANODE_STMT, QUERY_NODE_DROP_ANODE_STMT, QUERY_NODE_UPDATE_ANODE_STMT, + QUERY_NODE_ASSIGN_LEADER_STMT, // show statement nodes // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET' @@ -2653,6 +2654,15 @@ int32_t tSerializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq int32_t tDeserializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq); void tFreeSBalanceVgroupReq(SBalanceVgroupReq* pReq); +typedef struct { + int32_t useless; // useless + int32_t sqlLen; + char* sql; +} SAssignLeaderReq; + +int32_t tSerializeSAssignLeaderReq(void* buf, int32_t bufLen, SAssignLeaderReq* pReq); +int32_t tDeserializeSAssignLeaderReq(void* buf, int32_t bufLen, SAssignLeaderReq* pReq); +void tFreeSAssignLeaderReq(SAssignLeaderReq* pReq); typedef struct { int32_t vgId1; int32_t vgId2; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 9ea27485c2..4e65a4450f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -263,6 +263,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_CONFIG, "init-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_SDB, "config-sdb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESET_STREAM, "reset-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ASSIGN_LEADER, "assign-leader", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 26482a87d4..ea869f515e 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -681,6 +681,10 @@ typedef struct SBalanceVgroupStmt { ENodeType type; } SBalanceVgroupStmt; +typedef struct SAssignLeaderStmt { + ENodeType type; +} SAssignLeaderStmt; + typedef struct SBalanceVgroupLeaderStmt { ENodeType type; int32_t vgId; diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 7a51669d46..c19a885e8a 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -7644,6 +7644,46 @@ _exit: void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { FREESQL(); } +int32_t tSerializeSAssignLeaderReq(void *buf, int32_t bufLen, SAssignLeaderReq *pReq) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->useless)); + ENCODESQL(); + tEndEncode(&encoder); + +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAssignLeaderReq(void *buf, int32_t bufLen, SAssignLeaderReq *pReq) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + tDecoderInit(&decoder, buf, bufLen); + + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->useless)); + DECODESQL(); + tEndDecode(&decoder); + +_exit: + tDecoderClear(&decoder); + return code; +} + +void tFreeSAssignLeaderReq(SAssignLeaderReq *pReq) { FREESQL(); } + int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) { SEncoder encoder = {0}; int32_t code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 8f110dbcf3..cb2b5d6897 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -160,6 +160,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ASSIGN_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e20afb7201..6a3dbb488f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -46,6 +46,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq); +static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq); int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = { @@ -77,6 +78,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { // mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessVgroupBalanceLeaderMsg); mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP_LEADER, mndProcessVgroupBalanceLeaderMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ASSIGN_LEADER, mndProcessAssignLeaderMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -3570,6 +3572,30 @@ _OVER: TAOS_RETURN(code); } +static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq){ + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SArray *pArray = NULL; + void *pIter = NULL; + int64_t curMs = taosGetTimestampMs(); + + SAssignLeaderReq req = {0}; + if (tDeserializeSAssignLeaderReq(pReq->pCont, pReq->contLen, &req) != 0) { + code = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + auditRecord(pReq, pMnode->clusterId, "assignLeader", "", "", req.sql, req.sqlLen); + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("failed to assign leader since %s", tstrerror(code)); + } + + tFreeSAssignLeaderReq(&req); + TAOS_RETURN(code); +} + bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } bool mndVgroupInDnode(SVgObj *pVgroup, int32_t dnodeId) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index bfe86aa2ac..3e21aad20d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -199,6 +199,8 @@ const char* nodesNodeName(ENodeType type) { return "ResetStreamStmt"; case QUERY_NODE_BALANCE_VGROUP_STMT: return "BalanceVgroupStmt"; + case QUERY_NODE_ASSIGN_LEADER_STMT: + return "AssignLeaderStmt"; case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: return "BalanceVgroupLeaderStmt"; case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: @@ -8167,6 +8169,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return dropStreamStmtToJson(pObj, pJson); case QUERY_NODE_BALANCE_VGROUP_STMT: return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize. + case QUERY_NODE_ASSIGN_LEADER_STMT: + return TSDB_CODE_SUCCESS; case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to serialize. case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: @@ -8536,6 +8540,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDropStreamStmt(pJson, pObj); case QUERY_NODE_BALANCE_VGROUP_STMT: return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize. + case QUERY_NODE_ASSIGN_LEADER_STMT: + return TSDB_CODE_SUCCESS; case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: return TSDB_CODE_SUCCESS; case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index ae5b302d2d..c15753efc9 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -624,6 +624,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { case QUERY_NODE_BALANCE_VGROUP_STMT: code = makeNode(type, sizeof(SBalanceVgroupStmt), &pNode); break; + case QUERY_NODE_ASSIGN_LEADER_STMT: + code = makeNode(type, sizeof(SAssignLeaderStmt), &pNode); + break; case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: code = makeNode(type, sizeof(SBalanceVgroupLeaderStmt), &pNode); break; @@ -1497,6 +1500,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field case QUERY_NODE_RESET_STREAM_STMT: // no pointer field case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field + case QUERY_NODE_ASSIGN_LEADER_STMT: case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: // no pointer field case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 387bccf358..7807d519a9 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -309,6 +309,7 @@ SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STok SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt); +SNode* createAssingLeaderStmt(SAstCreateContext* pCxt); SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId); SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 439af13d71..e95fcb64bb 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -860,6 +860,9 @@ cmd ::= KILL COMPACT NK_INTEGER(A). /************************************************ merge/redistribute/ vgroup ******************************************/ cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } + +cmd ::= ASSIGN LEADER FORCE. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } + cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); } cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); } cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B). { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index c875cbad05..06a7c9d3a5 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -3873,7 +3873,7 @@ _err: SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); - SBalanceVgroupStmt* pStmt = NULL; + SAssignLeaderStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return (SNode*)pStmt; @@ -3881,6 +3881,16 @@ _err: return NULL; } +SNode* createAssignLeaderStmt(SAstCreateContext* pCxt) { + CHECK_PARSER_STATUS(pCxt); + SBalanceVgroupStmt* pStmt = NULL; + pCxt->errCode = nodesMakeNode(QUERY_NODE_ASSIGN_LEADER_STMT, (SNode**)&pStmt); + CHECK_MAKE_NODE(pStmt); + return (SNode*)pStmt; +_err: + return NULL; +} + SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId) { CHECK_PARSER_STATUS(pCxt); SBalanceVgroupLeaderStmt* pStmt = NULL; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 74dd1be614..48ebb7afb1 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8490,6 +8490,10 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) FILL_CMD_SQL(sql, sqlLen, pCmdReq, SBalanceVgroupReq, pReq); break; } + case TDMT_MND_ASSIGN_LEADER: { + FILL_CMD_SQL(sql, sqlLen, pCmdReq, SAssignLeaderReq, pReq); + break; + } case TDMT_MND_REDISTRIBUTE_VGROUP: { FILL_CMD_SQL(sql, sqlLen, pCmdReq, SRedistributeVgroupReq, pReq); break; @@ -12738,6 +12742,13 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm return code; } +static int32_t translateAssignLeader(STranslateContext* pCxt, SAssignLeaderStmt* pStmt) { + SAssignLeaderReq req = {0}; + int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP, (FSerializeFunc)tSerializeSAssignLeaderReq, &req); + tFreeSAssignLeaderReq(&req); + return code; +} + static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { SBalanceVgroupLeaderReq req = {0}; req.vgId = pStmt->vgId; @@ -13495,6 +13506,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_BALANCE_VGROUP_STMT: code = translateBalanceVgroup(pCxt, (SBalanceVgroupStmt*)pNode); break; + case QUERY_NODE_ASSIGN_LEADER_STMT: + code = translateAssignLeader(pCxt, (SAssignLeaderStmt*)pNode); + break; case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode); break; From 9608344833d6fdf9b3a59165d33750cd9b7c1cac Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 24 Jan 2025 03:47:45 +0000 Subject: [PATCH 02/71] feat/TS-5805-force-assign-leader-add-sql --- include/common/tmsgdef.h | 3 +-- source/libs/parser/inc/parAst.h | 2 +- source/libs/parser/inc/sql.y | 4 ++-- source/libs/parser/src/parAstCreater.c | 4 ++-- source/libs/parser/src/parTranslater.c | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 4e65a4450f..5ea23e2b0f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -187,7 +187,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_BATCH_META, "batch-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_CFG, "table-cfg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) - TD_DEF_MSG_TYPE(TDMT_MND_UNUSED1, "unused", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ASSIGN_LEADER, "assign-leader", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_TOPIC, "drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) @@ -263,7 +263,6 @@ TD_DEF_MSG_TYPE(TDMT_MND_CONFIG, "init-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_SDB, "config-sdb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESET_STREAM, "reset-stream", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ASSIGN_LEADER, "assign-leader", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 7807d519a9..d05438c04e 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -309,7 +309,7 @@ SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STok SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt); -SNode* createAssingLeaderStmt(SAstCreateContext* pCxt); +SNode* createAssignLeaderStmt(SAstCreateContext* pCxt); SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId); SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index e95fcb64bb..be425107c8 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -859,9 +859,9 @@ cmd ::= KILL TRANSACTION NK_INTEGER(A). cmd ::= KILL COMPACT NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_COMPACT_STMT, &A); } /************************************************ merge/redistribute/ vgroup ******************************************/ -cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } +cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } -cmd ::= ASSIGN LEADER FORCE. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } +cmd ::= ASSIGN LEADER FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); } cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 06a7c9d3a5..88cf1f3a2a 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -3873,7 +3873,7 @@ _err: SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); - SAssignLeaderStmt* pStmt = NULL; + SBalanceVgroupStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return (SNode*)pStmt; @@ -3883,7 +3883,7 @@ _err: SNode* createAssignLeaderStmt(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); - SBalanceVgroupStmt* pStmt = NULL; + SAssignLeaderStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ASSIGN_LEADER_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return (SNode*)pStmt; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 48ebb7afb1..08d2bf90bb 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12744,7 +12744,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateAssignLeader(STranslateContext* pCxt, SAssignLeaderStmt* pStmt) { SAssignLeaderReq req = {0}; - int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP, (FSerializeFunc)tSerializeSAssignLeaderReq, &req); + int32_t code = buildCmdMsg(pCxt, TDMT_MND_ASSIGN_LEADER, (FSerializeFunc)tSerializeSAssignLeaderReq, &req); tFreeSAssignLeaderReq(&req); return code; } From 2b4d4162a76962e5a508b15cecb2bb4b1fc40896 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 24 Jan 2025 08:50:04 +0000 Subject: [PATCH 03/71] feat/TS-5805-force-set-assign --- include/common/tmsg.h | 1 + source/common/src/msg/tmsg.c | 4 + source/dnode/mnode/impl/src/mndArbGroup.c | 78 ++++++++++++++- source/dnode/mnode/impl/src/mndVgroup.c | 26 ----- source/libs/sync/src/syncMain.c | 10 +- tests/army/cluster/arbitrator_restart.py | 111 ++++++++++++++++++++++ 6 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 tests/army/cluster/arbitrator_restart.py diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 670dc9453f..c1481cf716 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2578,6 +2578,7 @@ typedef struct { char* arbToken; int64_t arbTerm; char* memberToken; + int8_t force; } SVArbSetAssignedLeaderReq; int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq); diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index c19a885e8a..74425a296b 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -8210,6 +8210,7 @@ int32_t tSerializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSetA TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->arbToken)); TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->arbTerm)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->memberToken)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->force)); tEndEncode(&encoder); @@ -8239,6 +8240,9 @@ int32_t tDeserializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSe TAOS_CHECK_EXIT(terrno); } TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->memberToken)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->force)); + } tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 3879e8e6e2..88d0291e0d 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -46,6 +46,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp); static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp); static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextArbGroup(SMnode *pMnode, void *pIter); +static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq); static int32_t mndArbCheckToken(const char *token1, const char *token2) { if (token1 == NULL || token2 == NULL) return -1; @@ -71,6 +72,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp); mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp); + mndSetMsgHandle(pMnode, TDMT_MND_ASSIGN_LEADER, mndProcessAssignLeaderMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndRetrieveArbGroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup); @@ -530,11 +532,12 @@ static bool mndCheckArbMemberHbTimeout(SArbGroup *pArbGroup, int32_t index, int6 } static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, char *arbToken, int64_t arbTerm, - char *memberToken) { + char *memberToken, bool force) { SVArbSetAssignedLeaderReq req = {0}; req.arbToken = arbToken; req.arbTerm = arbTerm; req.memberToken = memberToken; + if (force) req.force = 1; int32_t reqLen = tSerializeSVArbSetAssignedLeaderReq(NULL, 0, &req); int32_t contLen = reqLen + sizeof(SMsgHead); @@ -554,10 +557,10 @@ static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, ch } static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken, - int64_t term, char *memberToken) { + int64_t term, char *memberToken, bool force) { int32_t code = 0; int32_t contLen = 0; - void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken); + void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken, force); if (!pHead) { mError("vgId:%d, failed to build set-assigned request", vgId); code = -1; @@ -643,6 +646,73 @@ void mndArbCheckSync(SArbGroup *pArbGroup, int64_t nowMs, ECheckSyncOp *pOp, SAr *pOp = CHECK_SYNC_UPDATE; } +static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1, lino = 0; + SArray *pArray = NULL; + void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + SArbGroup *pArbGroup = NULL; + + SAssignLeaderReq req = {0}; + if (tDeserializeSAssignLeaderReq(pReq->pCont, pReq->contLen, &req) != 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + mInfo("begin to process assign leader"); + + char arbToken[TSDB_ARB_TOKEN_SIZE]; + TAOS_CHECK_EXIT(mndGetArbToken(pMnode, arbToken)); + + int64_t term = mndGetTerm(pMnode); + if (term < 0) { + mError("arb failed to get term since %s", terrstr()); + code = -1; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + + while (1) { + pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup); + if (pIter == NULL) break; + + SArbGroup arbGroupDup = {0}; + + (void)taosThreadMutexLock(&pArbGroup->mutex); + mndArbGroupDupObj(pArbGroup, &arbGroupDup); + (void)taosThreadMutexUnlock(&pArbGroup->mutex); + + sdbRelease(pSdb, pArbGroup); + + int32_t dnodeId = 0; + for (int32_t i = 0; i < 2; i++) { + SDnodeObj *pDnode = mndAcquireDnode(pMnode, arbGroupDup.members[i].info.dnodeId); + bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); + mndReleaseDnode(pMnode, pDnode); + if (isonline) { + dnodeId = arbGroupDup.members[i].info.dnodeId; + break; + } + } + + (void)mndSendArbSetAssignedLeaderReq(pMnode, dnodeId, arbGroupDup.vgId, arbToken, term, "", true); + mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", arbGroupDup.vgId, dnodeId); + } + + code = 0; + + // auditRecord(pReq, pMnode->clusterId, "assignLeader", "", "", req.sql, req.sqlLen); + +_exit: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("failed to assign leader since %s", tstrerror(code)); + } + + tFreeSAssignLeaderReq(&req); + TAOS_RETURN(code); +} + static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { int32_t code = 0, lino = 0; SMnode *pMnode = pReq->info.node; @@ -695,7 +765,7 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { mTrace("vgId:%d, arb skip to send msg by check sync", vgId); break; case CHECK_SYNC_SET_ASSIGNED_LEADER: - (void)mndSendArbSetAssignedLeaderReq(pMnode, assgndDnodeId, vgId, arbToken, term, pAssgndLeader->token); + (void)mndSendArbSetAssignedLeaderReq(pMnode, assgndDnodeId, vgId, arbToken, term, pAssgndLeader->token, false); mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, assgndDnodeId); break; case CHECK_SYNC_CHECK_SYNC: diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 6a3dbb488f..e20afb7201 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -46,7 +46,6 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq); -static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq); int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = { @@ -78,7 +77,6 @@ int32_t mndInitVgroup(SMnode *pMnode) { // mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessVgroupBalanceLeaderMsg); mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP_LEADER, mndProcessVgroupBalanceLeaderMsg); - mndSetMsgHandle(pMnode, TDMT_MND_ASSIGN_LEADER, mndProcessAssignLeaderMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -3572,30 +3570,6 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq){ - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SArray *pArray = NULL; - void *pIter = NULL; - int64_t curMs = taosGetTimestampMs(); - - SAssignLeaderReq req = {0}; - if (tDeserializeSAssignLeaderReq(pReq->pCont, pReq->contLen, &req) != 0) { - code = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - auditRecord(pReq, pMnode->clusterId, "assignLeader", "", "", req.sql, req.sqlLen); - -_OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("failed to assign leader since %s", tstrerror(code)); - } - - tFreeSAssignLeaderReq(&req); - TAOS_RETURN(code); -} - bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } bool mndVgroupInDnode(SVgObj *pVgroup, int32_t dnodeId) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0933fd48c7..ea91412536 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -334,10 +334,12 @@ int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg) { ths->arbTerm = TMAX(req.arbTerm, ths->arbTerm); - if (strncmp(req.memberToken, ths->arbToken, TSDB_ARB_TOKEN_SIZE) != 0) { - sInfo("vgId:%d, skip to set assigned leader, token mismatch, local:%s, msg:%s", ths->vgId, ths->arbToken, - req.memberToken); - goto _OVER; + if (!req.force) { + if (strncmp(req.memberToken, ths->arbToken, TSDB_ARB_TOKEN_SIZE) != 0) { + sInfo("vgId:%d, skip to set assigned leader, token mismatch, local:%s, msg:%s", ths->vgId, ths->arbToken, + req.memberToken); + goto _OVER; + } } if (ths->state != TAOS_SYNC_STATE_ASSIGNED_LEADER) { diff --git a/tests/army/cluster/arbitrator_restart.py b/tests/army/cluster/arbitrator_restart.py new file mode 100644 index 0000000000..7a02e46630 --- /dev/null +++ b/tests/army/cluster/arbitrator_restart.py @@ -0,0 +1,111 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame import epath +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + updatecfgDict = {'dDebugFlag':131} + super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + + self.valgrind = 0 + self.db = "test" + self.stb = "meters" + self.childtable_count = 10 + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;') + + time.sleep(1) + + tdSql.execute("use db;") + + tdSql.execute("CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);") + + tdSql.execute("CREATE TABLE d0 USING meters TAGS (\"California.SanFrancisco\", 2);"); + + count = 0 + + while count < 100: + tdSql.query("show arbgroups;") + + if tdSql.getData(0, 4) == 1: + break + + tdLog.info("wait 1 seconds for is sync") + time.sleep(1) + + count += 1 + + sc.dnodeStop(2) + sc.dnodeStop(3) + + sc.dnodeStart(2) + + count = 0 + while count < 100: + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "candidate") or (tdSql.getData(0, 6) == "candidate"): + break + + tdLog.info("wait 1 seconds for candidate") + time.sleep(1) + + count += 1 + + tdSql.execute("balance vgroup;") + + count = 0 + while count < 100: + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 6) == "assigned "): + break + + tdLog.info("wait 1 seconds for set assigned") + time.sleep(1) + + count += 1 + + tdSql.execute("INSERT INTO d0 VALUES (NOW, 10.3, 219, 0.31);") + + sc.dnodeStart(3) + + count = 0 + + while count < 100: + tdSql.query("show arbgroups;") + + if tdSql.getData(0, 4) == 1: + break + + tdLog.info("wait 1 seconds for is sync") + time.sleep(1) + + count += 1 + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From b07f4a9bacbb087b65c4d20163efaf70ada02904 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 24 Jan 2025 09:08:47 +0000 Subject: [PATCH 04/71] feat/TS-5805-force-assign-leader-change-sql --- source/libs/parser/inc/sql.y | 4 ++-- tests/army/cluster/arbitrator_restart.py | 2 +- tests/parallel_test/cases.task | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index be425107c8..4ed05d6b74 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -859,9 +859,9 @@ cmd ::= KILL TRANSACTION NK_INTEGER(A). cmd ::= KILL COMPACT NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_COMPACT_STMT, &A); } /************************************************ merge/redistribute/ vgroup ******************************************/ -cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } +cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } -cmd ::= ASSIGN LEADER FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } +cmd ::= BALANCE VGROUP FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); } cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); } diff --git a/tests/army/cluster/arbitrator_restart.py b/tests/army/cluster/arbitrator_restart.py index 7a02e46630..21b7a5b82b 100644 --- a/tests/army/cluster/arbitrator_restart.py +++ b/tests/army/cluster/arbitrator_restart.py @@ -71,7 +71,7 @@ class TDTestCase(TBase): count += 1 - tdSql.execute("balance vgroup;") + tdSql.execute("BALANCE VGROUP FORCE;") count = 0 while count < 100: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0201c88d2b..a38b0cd45e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -22,6 +22,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py -N 3 -M 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/arbitrator.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f cluster/arbitrator_restart.py -N 3 ,,n,army,python3 ./test.py -f storage/s3/s3Basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py From b422e2b2f25942f79da1ed96a7d7681d80f31bd9 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 12 Feb 2025 06:29:24 +0000 Subject: [PATCH 05/71] feat/TS-5805-force-assign-leader-msg-type --- include/common/tmsgdef.h | 3 ++- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mnode/impl/src/mndArbGroup.c | 2 +- source/libs/parser/src/parTranslater.c | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 5ea23e2b0f..29a7e52482 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -187,7 +187,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_BATCH_META, "batch-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_CFG, "table-cfg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) - TD_DEF_MSG_TYPE(TDMT_MND_ASSIGN_LEADER, "assign-leader", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_UNUSED1, "unused", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_TOPIC, "drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) @@ -421,6 +421,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ARB_ASSIGN_LEADER, "mnd-arb-assign-leader", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_MND_ARB_MSG) TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index cb2b5d6897..46b0877476 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -160,7 +160,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ASSIGN_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ARB_ASSIGN_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 2e5d20fb68..48c7dace16 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -71,7 +71,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp); mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp); - mndSetMsgHandle(pMnode, TDMT_MND_ASSIGN_LEADER, mndProcessAssignLeaderMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ARB_ASSIGN_LEADER, mndProcessAssignLeaderMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndRetrieveArbGroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 756aa97a5f..474260a81f 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8532,7 +8532,7 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) FILL_CMD_SQL(sql, sqlLen, pCmdReq, SBalanceVgroupReq, pReq); break; } - case TDMT_MND_ASSIGN_LEADER: { + case TDMT_MND_ARB_ASSIGN_LEADER: { FILL_CMD_SQL(sql, sqlLen, pCmdReq, SAssignLeaderReq, pReq); break; } @@ -12786,7 +12786,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateAssignLeader(STranslateContext* pCxt, SAssignLeaderStmt* pStmt) { SAssignLeaderReq req = {0}; - int32_t code = buildCmdMsg(pCxt, TDMT_MND_ASSIGN_LEADER, (FSerializeFunc)tSerializeSAssignLeaderReq, &req); + int32_t code = buildCmdMsg(pCxt, TDMT_MND_ARB_ASSIGN_LEADER, (FSerializeFunc)tSerializeSAssignLeaderReq, &req); tFreeSAssignLeaderReq(&req); return code; } From 977372d376411867e21dc6800cf342f678f2fd3c Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 12 Feb 2025 06:50:37 +0000 Subject: [PATCH 06/71] feat/TS-5805-force-assign-leader-fix-cases --- source/libs/parser/inc/sql.y | 2 +- tests/army/cluster/arbitrator_restart.py | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 796ee76606..341a699288 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -863,7 +863,7 @@ cmd ::= KILL COMPACT NK_INTEGER(A). /************************************************ merge/redistribute/ vgroup ******************************************/ cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } -cmd ::= BALANCE VGROUP FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } +cmd ::= BALANCE LEADER FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); } cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); } diff --git a/tests/army/cluster/arbitrator_restart.py b/tests/army/cluster/arbitrator_restart.py index 21b7a5b82b..4a80aa1c5a 100644 --- a/tests/army/cluster/arbitrator_restart.py +++ b/tests/army/cluster/arbitrator_restart.py @@ -46,14 +46,17 @@ class TDTestCase(TBase): while count < 100: tdSql.query("show arbgroups;") - if tdSql.getData(0, 4) == 1: + if tdSql.getData(0, 4) == True: break tdLog.info("wait 1 seconds for is sync") time.sleep(1) count += 1 - + if count == 100: + tdLog.exit("arbgroup sync failed") + return + sc.dnodeStop(2) sc.dnodeStop(3) @@ -70,8 +73,11 @@ class TDTestCase(TBase): time.sleep(1) count += 1 - - tdSql.execute("BALANCE VGROUP FORCE;") + if count == 100: + tdLog.exit("wait candidate failed") + return + + tdSql.execute("BALANCE LEADER FORCE;") count = 0 while count < 100: @@ -84,6 +90,9 @@ class TDTestCase(TBase): time.sleep(1) count += 1 + if count == 100: + tdLog.exit("assign leader failed") + return tdSql.execute("INSERT INTO d0 VALUES (NOW, 10.3, 219, 0.31);") @@ -101,6 +110,9 @@ class TDTestCase(TBase): time.sleep(1) count += 1 + if count == 100: + tdLog.exit("arbgroup sync failed") + return def stop(self): tdSql.close() From ee0c280fe6ed3fe4875349f7c8ddc961e0422c5b Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 12 Feb 2025 08:20:56 +0000 Subject: [PATCH 07/71] feat/TS-5805-force-assign-leader-token --- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/parTokenizer.c | 1 + tests/army/cluster/arbitrator_restart.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 341a699288..2bc9e8a668 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -863,7 +863,7 @@ cmd ::= KILL COMPACT NK_INTEGER(A). /************************************************ merge/redistribute/ vgroup ******************************************/ cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } -cmd ::= BALANCE LEADER FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } +cmd ::= ASSIGN LEADER FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); } cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); } cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 3b08d403dc..300af653af 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -359,6 +359,7 @@ static SKeyword keywordTable[] = { {"ON_FAILURE", TK_ON_FAILURE}, {"NOTIFY_HISTORY", TK_NOTIFY_HISTORY}, {"REGEXP", TK_REGEXP}, + {"ASSIGN", TK_ASSIGN}, }; // clang-format on diff --git a/tests/army/cluster/arbitrator_restart.py b/tests/army/cluster/arbitrator_restart.py index 4a80aa1c5a..aad51e7d26 100644 --- a/tests/army/cluster/arbitrator_restart.py +++ b/tests/army/cluster/arbitrator_restart.py @@ -77,7 +77,7 @@ class TDTestCase(TBase): tdLog.exit("wait candidate failed") return - tdSql.execute("BALANCE LEADER FORCE;") + tdSql.execute("ASSIGN LEADER FORCE;") count = 0 while count < 100: From 80183ad8f3d81214544040acdafbec29f0137934 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 20 Feb 2025 11:11:27 +0800 Subject: [PATCH 08/71] enh: grant support TDgpt --- include/common/tgrant.h | 1 + include/util/taoserror.h | 1 + source/util/src/terror.c | 1 + source/util/test/errorCodeTable.ini | 1 + tests/system-test/0-others/information_schema.py | 1 + 5 files changed, 5 insertions(+) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 1660cc6a1b..17a383f7c6 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -61,6 +61,7 @@ typedef enum { TSDB_GRANT_ACTIVE_ACTIVE, TSDB_GRANT_DUAL_REPLICA_HA, TSDB_GRANT_DB_ENCRYPTION, + TSDB_GRANT_TD_GPT, } EGrantType; int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 397118411c..c93e1ffde4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -693,6 +693,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082B) #define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C) #define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082D) +#define TSDB_CODE_GRANT_TD_GPT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082E) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f34b00bec5..7e32a198e4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -564,6 +564,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED, "License expired for m TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED, "License expired for database encryption function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TD_GPT_EXPIRED, "License expired for TDgpt function") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") diff --git a/source/util/test/errorCodeTable.ini b/source/util/test/errorCodeTable.ini index f67c8ab834..717499ca75 100644 --- a/source/util/test/errorCodeTable.ini +++ b/source/util/test/errorCodeTable.ini @@ -400,6 +400,7 @@ TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED = 0x8000082A TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED = 0x8000082B TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED = 0x8000082C TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED = 0x8000082D +TSDB_CODE_GRANT_TD_GPT_EXPIRED = 0x8000082E TSDB_CODE_SYN_TIMEOUT = 0x80000903 TSDB_CODE_SYN_MISMATCHED_SIGNATURE = 0x80000907 TSDB_CODE_SYN_NOT_LEADER = 0x8000090C diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 96cd596a20..63f0a236c3 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -283,6 +283,7 @@ class TDTestCase: 'active_active':'Active-Active', 'dual_replica':'Dual-Replica HA', 'db_encryption':'Database Encryption', + 'tdgpt':'TDgpt', 'opc_da':'OPC_DA', 'opc_ua':'OPC_UA', 'pi':'Pi', From 29129b6b3a18c275b4d5159c51b7a04019b2be95 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 20 Feb 2025 12:59:09 +0800 Subject: [PATCH 09/71] chore: ignore error check for 0x8000082E --- tests/army/whole/checkErrorCode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py index 1a14269f54..d66ae56e46 100644 --- a/tests/army/whole/checkErrorCode.py +++ b/tests/army/whole/checkErrorCode.py @@ -50,7 +50,7 @@ ignoreCodes = [ '0x80000734', '0x80000735', '0x80000736', '0x80000737', '0x80000738', '0x8000080E', '0x8000080F', '0x80000810', '0x80000811', '0x80000812', '0x80000813', '0x80000814', '0x80000815', '0x80000816', '0x80000817', '0x80000818', '0x80000819', '0x80000820', '0x80000821', '0x80000822', '0x80000823', '0x80000824', '0x80000825', '0x80000826', '0x80000827', '0x80000828', '0x80000829', '0x8000082A', '0x8000082B', '0x8000082C', - '0x8000082D', '0x80000907', '0x80000919', '0x8000091A', '0x8000091B', '0x8000091C', '0x8000091D', '0x8000091E', '0x8000091F', '0x80000A00', + '0x8000082D', '0x8000082E', '0x80000907', '0x80000919', '0x8000091A', '0x8000091B', '0x8000091C', '0x8000091D', '0x8000091E', '0x8000091F', '0x80000A00', '0x80000A01', '0x80000A03', '0x80000A06', '0x80000A07', '0x80000A08', '0x80000A09', '0x80000A0A', '0x80000A0B', '0x80000A0E', '0x80002206', '0x80002207', '0x80002406', '0x80002407', '0x80002503', '0x80002506', '0x80002507', '0x8000261B', '0x80002653', '0x80002668', '0x80002669', '0x8000266A', '0x8000266B', '0x8000266C', '0x8000266D', '0x8000266E', '0x8000266F', '0x80002670', '0x80002671', '0x80002672', '0x80002673', From 078c7bdc31276e7aa5f4a055a6e1b05889dc8d0c Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 20 Feb 2025 17:33:48 +0800 Subject: [PATCH 10/71] fix/TS-6028-check-column-name --- source/dnode/mnode/impl/src/mndStb.c | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d46968a22d..7ce970403f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1259,6 +1259,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SDbObj *pDb = NULL; SMCreateStbReq createReq = {0}; bool isAlter = false; + SHashObj *pHash = NULL; if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { code = TSDB_CODE_INVALID_MSG; @@ -1319,6 +1320,25 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } + pHash = taosHashInit(3, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + void *pIter = NULL; + + for (int32_t i = 0; i < createReq.numOfColumns; ++i) { + SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i); + if (taosHashPut(pHash, pField->name, sizeof(pField->name), pField->name, sizeof(pField->name)) != 0) { + code = TSDB_CODE_TSC_DUP_COL_NAMES; + goto _OVER; + } + } + + for (int32_t i = 0; i < createReq.numOfTags; ++i) { + SField *pField = taosArrayGet(createReq.pTags, i); + if (taosHashPut(pHash, pField->name, sizeof(pField->name), pField->name, sizeof(pField->name)) != 0) { + code = TSDB_CODE_TSC_DUP_COL_NAMES; + goto _OVER; + } + } + pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { code = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -1383,6 +1403,17 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSMCreateStbReq(&createReq); + if (pHash != NULL) { + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq *pDb = (STablesReq *)pIter; + taosArrayDestroy(pDb->pTables); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); + } + TAOS_RETURN(code); } From 12c030c5782b5fe06ebfca42d163f8d7f9f8ab92 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Feb 2025 17:51:22 +0800 Subject: [PATCH 11/71] fix: stmt with sql write make consistency --- tools/taos-tools/src/benchData.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tools/taos-tools/src/benchData.c b/tools/taos-tools/src/benchData.c index f5fdccf538..9b58ad06ca 100644 --- a/tools/taos-tools/src/benchData.c +++ b/tools/taos-tools/src/benchData.c @@ -812,17 +812,6 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, continue; } } - if (field->type == TSDB_DATA_TYPE_TIMESTAMP && !tag) { - n = snprintf(sampleDataBuf + pos, bufLen - pos, "now,"); - if (n < 0 || n >= bufLen - pos) { - errorPrint("%s() LN%d snprintf overflow\n", - __func__, __LINE__); - return -1; - } else { - pos += n; - continue; - } - } switch (field->type) { case TSDB_DATA_TYPE_BOOL: { bool boolTmp = tmpBool(field); From 7b5edade957dd3c2d5348637ecf7d164a459eb8d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Feb 2025 19:50:37 +0800 Subject: [PATCH 12/71] fix:[TD-337958]send batch metadata & remove data if subscribe only meta --- source/dnode/vnode/src/tq/tqRead.c | 16 +++ source/dnode/vnode/src/tq/tqScan.c | 4 +- source/dnode/vnode/src/tq/tqUtil.c | 171 +++++++++++++++++++---------- 3 files changed, 134 insertions(+), 57 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index d650ae9751..ee705ed374 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1098,6 +1098,22 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* block *pSubmitTbDataRet = pSubmitTbData; } + if (fetchMeta == ONLY_META) { + if (pSubmitTbData->pCreateTbReq != NULL) { + if (pRsp->createTableReq == NULL){ + pRsp->createTableReq = taosArrayInit(0, POINTER_BYTES); + if (pRsp->createTableReq == NULL){ + return terrno; + } + } + if (taosArrayPush(pRsp->createTableReq, &pSubmitTbData->pCreateTbReq) == NULL){ + return terrno; + } + pSubmitTbData->pCreateTbReq = NULL; + } + return 0; + } + int32_t sversion = pSubmitTbData->sver; int64_t uid = pSubmitTbData->uid; pReader->lastBlkUid = uid; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 70a165906e..9d1b99f483 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -339,6 +339,9 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int bool tmp = (pSubmitTbData->flags & pRequest->sourceExcluded) != 0; TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); + if (pHandle->fetchMeta == ONLY_META){ + goto END; + } int32_t blockNum = taosArrayGetSize(pBlocks) == 0 ? 1 : taosArrayGetSize(pBlocks); if (pRsp->withTbName) { @@ -347,7 +350,6 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int TSDB_CHECK_CODE(code, lino, END); } - tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbData->pCreateTbReq == NULL); TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); for (int32_t i = 0; i < blockNum; i++) { if (taosArrayGetSize(pBlocks) == 0){ diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 9866528446..4d908b5d57 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -226,6 +226,82 @@ static void tDeleteCommon(void* parm) {} taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : \ (pRequest->rawData == 1 ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP) +static int32_t buildBatchMeta(SMqBatchMetaRsp *btMetaRsp, int16_t type, int32_t bodyLen, void* body){ + int32_t code = 0; + + if (!btMetaRsp->batchMetaReq) { + btMetaRsp->batchMetaReq = taosArrayInit(4, POINTER_BYTES); + TQ_NULL_GO_TO_END(btMetaRsp->batchMetaReq); + btMetaRsp->batchMetaLen = taosArrayInit(4, sizeof(int32_t)); + TQ_NULL_GO_TO_END(btMetaRsp->batchMetaLen); + } + + SMqMetaRsp tmpMetaRsp = {0}; + tmpMetaRsp.resMsgType = type; + tmpMetaRsp.metaRspLen = bodyLen; + tmpMetaRsp.metaRsp = body; + uint32_t len = 0; + tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code); + if (TSDB_CODE_SUCCESS != code) { + tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); + goto END; + } + int32_t tLen = sizeof(SMqRspHead) + len; + void* tBuf = taosMemoryCalloc(1, tLen); + TQ_NULL_GO_TO_END(tBuf); + void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); + SEncoder encoder = {0}; + tEncoderInit(&encoder, metaBuff, len); + code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp); + tEncoderClear(&encoder); + + if (code < 0) { + tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); + goto END; + } + TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp->batchMetaReq, &tBuf)); + TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp->batchMetaLen, &tLen)); + +END: + return code; +} + +static int32_t buildBatchCreateContent(SMqDataRsp *taosxRsp, void** pBuf, int32_t *len){ + int32_t code = 0; + SVCreateTbBatchReq pReq = {0}; + pReq.nReqs = taosArrayGetSize(taosxRsp->createTableReq); + pReq.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); + TQ_NULL_GO_TO_END(pReq.pArray); + for (int i = 0; i < taosArrayGetSize(taosxRsp->createTableReq); i++){ + void *createTableReq = taosArrayGetP(taosxRsp->createTableReq, i); + TQ_NULL_GO_TO_END(taosArrayPush(pReq.pArray, createTableReq)); + } + tEncodeSize(tEncodeSVCreateTbBatchReq, &pReq, *len, code); + if (code < 0) { + goto END; + } + len += sizeof(SMsgHead); + *pBuf = taosMemoryMalloc(*len); + TQ_NULL_GO_TO_END(pBuf); + SEncoder coder = {0}; + tEncoderInit(&coder, POINTER_SHIFT(*pBuf, sizeof(SMsgHead)), *len); + code = tEncodeSVCreateTbBatchReq(&coder, &pReq); + tEncoderClear(&coder); + +END: + taosArrayDestroy(pReq.pArray); + return code; +} + +#define SEND_BATCH_META_RSP \ +tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);\ +code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);\ +goto END; + +#define SEND_DATA_RSP \ +tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);\ +code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, POLL_RSP_TYPE(pRequest, taosxRsp), vgId);\ +goto END; static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int32_t vgId = TD_VID(pTq->pVnode); @@ -261,6 +337,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, uint64_t st = taosGetTimestampMs(); int totalRows = 0; int32_t totalMetaRows = 0; + int32_t totalAutoMetaRows = 0; while (1) { int32_t savedEpoch = atomic_load_32(&pHandle->epoch); if (savedEpoch > pRequest->epoch) { @@ -271,15 +348,10 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) { - if (totalMetaRows > 0) { - tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); - code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); - goto END; + if (totalMetaRows > 0 || totalAutoMetaRows > 0) { + SEND_BATCH_META_RSP } - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - POLL_RSP_TYPE(pRequest, taosxRsp), vgId); - goto END; + SEND_DATA_RSP } SWalCont* pHead = &pHandle->pWalReader->pHead->head; @@ -288,11 +360,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { + if (totalAutoMetaRows > 0){ + SEND_BATCH_META_RSP + } if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - POLL_RSP_TYPE(pRequest, taosxRsp), vgId); - goto END; + SEND_DATA_RSP } if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) { @@ -318,53 +390,20 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); goto END; } - - if (!btMetaRsp.batchMetaReq) { - btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES); - TQ_NULL_GO_TO_END(btMetaRsp.batchMetaReq); - btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t)); - TQ_NULL_GO_TO_END(btMetaRsp.batchMetaLen); - } + code = buildBatchMeta(&btMetaRsp, pHead->msgType, pHead->bodyLen, pHead->body); fetchVer++; - - SMqMetaRsp tmpMetaRsp = {0}; - tmpMetaRsp.resMsgType = pHead->msgType; - tmpMetaRsp.metaRspLen = pHead->bodyLen; - tmpMetaRsp.metaRsp = pHead->body; - uint32_t len = 0; - tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code); - if (TSDB_CODE_SUCCESS != code) { - tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); + if (code != 0){ continue; } - int32_t tLen = sizeof(SMqRspHead) + len; - void* tBuf = taosMemoryCalloc(1, tLen); - TQ_NULL_GO_TO_END(tBuf); - void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); - SEncoder encoder = {0}; - tEncoderInit(&encoder, metaBuff, len); - code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp); - tEncoderClear(&encoder); - - if (code < 0) { - tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); - continue; - } - TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf)); - TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaLen, &tLen)); totalMetaRows++; if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > pRequest->timeout)) { - tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); - code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); - goto END; + SEND_BATCH_META_RSP } continue; } if (totalMetaRows > 0) { - tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); - code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); - goto END; + SEND_BATCH_META_RSP } // process data @@ -376,17 +415,37 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest)); + if (pHandle->fetchMeta == ONLY_META){ + int32_t len = 0; + void *pBuf = NULL; + code = buildBatchCreateContent(&taosxRsp, &pBuf, &len); + if (code == 0){ + code = buildBatchMeta(&btMetaRsp, TDMT_VND_CREATE_TABLE, len, pBuf); + } + taosMemoryFree(pBuf); + taosArrayDestroyP(taosxRsp.createTableReq, NULL); + taosxRsp.createTableReq = NULL; + fetchVer++; + if (code != 0){ + continue; + } + totalAutoMetaRows ++; + if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > pRequest->timeout)) { + SEND_BATCH_META_RSP + } + continue; + } + if ((pRequest->rawData == 0 && totalRows >= pRequest->minPollRows) || (taosGetTimestampMs() - st > pRequest->timeout) || (pRequest->rawData != 0 && (taosArrayGetSize(taosxRsp.blockData) > pRequest->minPollRows || terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT))) { -// tqDebug("start to send rsp, block num:%d, totalRows:%d, createTableNum:%d, terrno:%d", -// (int)taosArrayGetSize(taosxRsp.blockData), totalRows, taosxRsp.createTableNum, terrno); - tqOffsetResetToLog(&taosxRsp.rspOffset, terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT ? fetchVer : fetchVer + 1); - code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - POLL_RSP_TYPE(pRequest, taosxRsp), vgId); - if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){terrno = 0;} - goto END; + if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){ + terrno = 0; + } else{ + fetchVer++; + } + SEND_DATA_RSP } else { fetchVer++; } From a75aa42beccc230ba87e61e9062978411764a2df Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Feb 2025 21:53:52 +0800 Subject: [PATCH 13/71] fix: add new case taosCli.py --- tests/army/cmdline/json/taosCli.json | 55 ++++++++++++++--------- tests/army/cmdline/json/taosCliDesc.json | 57 ++++++++++++++++++++++++ tests/army/cmdline/taosCli.py | 34 +++++++++++++- tests/army/frame/caseBase.py | 8 ++++ 4 files changed, 132 insertions(+), 22 deletions(-) create mode 100644 tests/army/cmdline/json/taosCliDesc.json diff --git a/tests/army/cmdline/json/taosCli.json b/tests/army/cmdline/json/taosCli.json index 2dcd28afdb..5bdecef72b 100644 --- a/tests/army/cmdline/json/taosCli.json +++ b/tests/army/cmdline/json/taosCli.json @@ -23,32 +23,45 @@ "childtable_prefix":"d", "data_source":"rand", "insert_mode":"taosc", - "childtable_count": 1, - "insert_rows":10, + "childtable_count": 2, + "insert_rows":100, "timestamp_step":1000, "start_timestamp":"2022-10-01 00:00:00.000", "columns":[ - { "type": "int", "count": 1000, "max": 2000, "min": 0 } + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} ], "tags":[ - { - "type":"binary", - "name":"location", - "max":64, - "min":1, - "values":[ - "San Francisco", - "Los Angles", - "San Diego", - "San Jose", - "Palo Alto", - "Campbell", - "Mountain View", - "Sunnyvale", - "Santa Clara", - "Cupertino" - ] - } + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} ] } ] diff --git a/tests/army/cmdline/json/taosCliDesc.json b/tests/army/cmdline/json/taosCliDesc.json new file mode 100644 index 0000000000..2dcd28afdb --- /dev/null +++ b/tests/army/cmdline/json/taosCliDesc.json @@ -0,0 +1,57 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 1, + "insert_rows":10, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "int", "count": 1000, "max": 2000, "min": 0 } + ], + "tags":[ + { + "type":"binary", + "name":"location", + "max":64, + "min":1, + "values":[ + "San Francisco", + "Los Angles", + "San Diego", + "San Jose", + "Palo Alto", + "Campbell", + "Mountain View", + "Sunnyvale", + "Santa Clara", + "Cupertino" + ] + } + ] + } + ] + } + ] +} diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index d446a25c86..21006f8d1c 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -42,7 +42,7 @@ class TDTestCase(TBase): tdLog.info(f"check describe show full.") # insert - json = "cmdline/json/taosCli.json" + json = "cmdline/json/taosCliDesc.json" db, stb, childCount, insertRows = self.insertBenchJson(json) # describe sql = f"describe {db}.{stb};" @@ -53,13 +53,45 @@ class TDTestCase(TBase): tdSql.query(sql) tdSql.checkRows(2 + 1000) + def checkBasic(self): + tdLog.info(f"check describe show full.") + + # insert + json = "cmdline/json/taosCli.json" + db, stb, childCount, insertRows = self.insertBenchJson(json) + + result = "Query OK, 10 row(s)" + + + # hori + cmd = f'taos -s "select * {db}.{stb} limit 10"' + rlist = etool.runBinFile("taos", cmd) + # line count + self.checkSame(len(rlist), 18) + # last line + self.checkSame(rlist[-1][:len(result)], result) + + # vec + rlist = etool.runBinFile("taos", cmd + "\G") + # line count + self.checkSame(len(rlist), 346) + self.checkSame(rlist[310], "************************** 10.row ***************************") + # last line + self.checkSame(rlist[-1][:len(result)], result) + # run def run(self): tdLog.debug(f"start to excute {__file__}") + # check basic + self.checkBasic() + # check show whole self.checkDescribe() + # full types show + + tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index d8dcb29e44..55fc5aa4c7 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -231,6 +231,14 @@ class TBase: tdLog.info("sql1 same result with sql2.") + # check same value + def checkSame(self, real, expect, show = True): + if real == expect: + if show: + tdLog.info(f"check same succ. real={real} expect={expect}.") + else: + tdLog.exit(f"check same failed. real={real} expect={expect}.") + # # get db information # From 7211940b89bac98a477fa13acf6270a2e462e3ed Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Feb 2025 21:57:04 +0800 Subject: [PATCH 14/71] fix: remove duplicate taos --- tests/army/cmdline/taosCli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 21006f8d1c..8e604c0686 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -64,7 +64,7 @@ class TDTestCase(TBase): # hori - cmd = f'taos -s "select * {db}.{stb} limit 10"' + cmd = f'-s "select * {db}.{stb} limit 10"' rlist = etool.runBinFile("taos", cmd) # line count self.checkSame(len(rlist), 18) From 9e865a4e19f426233fc14721da01142ad4fb5bb0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Feb 2025 22:06:35 +0800 Subject: [PATCH 15/71] fix: case passed ok --- tests/army/cmdline/taosCli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 8e604c0686..35cceec933 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -64,20 +64,20 @@ class TDTestCase(TBase): # hori - cmd = f'-s "select * {db}.{stb} limit 10"' - rlist = etool.runBinFile("taos", cmd) + cmd = f'-s "select * from {db}.{stb} limit 10' + rlist = etool.runBinFile("taos", cmd + '"') # line count self.checkSame(len(rlist), 18) # last line - self.checkSame(rlist[-1][:len(result)], result) + self.checkSame(rlist[-2][:len(result)], result) # vec - rlist = etool.runBinFile("taos", cmd + "\G") + rlist = etool.runBinFile("taos", cmd + '\G"') # line count self.checkSame(len(rlist), 346) - self.checkSame(rlist[310], "************************** 10.row ***************************") + self.checkSame(rlist[310], "*************************** 10.row ***************************") # last line - self.checkSame(rlist[-1][:len(result)], result) + self.checkSame(rlist[-2][:len(result)], result) # run def run(self): From aae88d7cc3db6119fca1eb489211933761a2ed9b Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 21 Feb 2025 04:10:52 +0800 Subject: [PATCH 16/71] chore: adjust test case for grant TDgpt --- tests/system-test/0-others/grant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/grant.py b/tests/system-test/0-others/grant.py index 25af6eb842..038992b65a 100644 --- a/tests/system-test/0-others/grant.py +++ b/tests/system-test/0-others/grant.py @@ -237,7 +237,7 @@ class TDTestCase: tdLog.info(f"expireTime: {expireTime}, serviceTime: {serviceTime}") tdSql.checkEqual(True, abs(expireTime - serviceTime - 864000) < 15) tdSql.query(f'show grants full;') - nGrantItems = 31 + nGrantItems = 32 tdSql.checkEqual(len(tdSql.queryResult), nGrantItems) tdSql.checkEqual(tdSql.queryResult[0][2], serviceTimeStr) for i in range(1, nGrantItems): From 003732cd322cfd4d66d8494ee8db58d0772144a5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 21 Feb 2025 09:25:16 +0800 Subject: [PATCH 17/71] fix: read exe return result with output to file --- tests/army/frame/eos.py | 44 +++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 802b62e052..363714a6d1 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -64,12 +64,23 @@ def exeNoWait(file): return exe(cmd) # run return output and error -def run(command, timeout = 10): - process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - process.wait(timeout) +def run(command, show=True): + # out to file + id = time.clock_gettime_ns(time.CLOCK_REALTIME) % 100000 + out = f"out_{id}.txt" + err = f"err_{id}.txt" + + ret = exe(command + f" 1>{out} 2>{err}") - output = process.stdout.read().decode(encoding="gbk") - error = process.stderr.read().decode(encoding="gbk") + # read from file + output = readFileContext(out) + error = readFileContext(err) + + # del + if os.path.exists(out): + os.remove(out) + if os.path.exists(err): + os.remove(err) return output, error @@ -84,4 +95,25 @@ def runRetList(command, timeout=10): # def delFile(file): - return exe(f"rm -rf {file}") \ No newline at end of file + return exe(f"rm -rf {file}") + +def readFileContext(filename): + file = open(filename) + context = file.read() + file.close() + return context + +def writeFileContext(filename, context): + file = open(filename, "w") + file.write(context) + file.close() + +def appendFileContext(filename, context): + global resultContext + resultContext += context + try: + file = open(filename, "a") + wsize = file.write(context) + file.close() + except: + print(f"appand file error context={context} .") \ No newline at end of file From cdb11dd49774284a6f2316822bf99da563fafa1d Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 21 Feb 2025 10:10:44 +0800 Subject: [PATCH 18/71] fix/TS-6028-column-name-fix-cases --- source/dnode/mnode/impl/src/mndStb.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7ce970403f..5ada3e42e3 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1404,13 +1404,6 @@ _OVER: tFreeSMCreateStbReq(&createReq); if (pHash != NULL) { - pIter = taosHashIterate(pHash, NULL); - while (pIter) { - STablesReq *pDb = (STablesReq *)pIter; - taosArrayDestroy(pDb->pTables); - pIter = taosHashIterate(pHash, pIter); - } - taosHashCleanup(pHash); } From 8efc8d85f032336e4e7db73bff78dd2ddced0605 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 21 Feb 2025 11:10:49 +0800 Subject: [PATCH 19/71] fix/TD-33804-fix-ci-add-check-status --- .../3-enterprise/restore/kill_restore_dnode.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/system-test/3-enterprise/restore/kill_restore_dnode.py b/tests/system-test/3-enterprise/restore/kill_restore_dnode.py index d29a11f159..4d0dca1952 100644 --- a/tests/system-test/3-enterprise/restore/kill_restore_dnode.py +++ b/tests/system-test/3-enterprise/restore/kill_restore_dnode.py @@ -75,6 +75,14 @@ class TDTestCase: def restoreDnodeThread(self, p, newTdSql): sleep(1) + count = 0 + while count < 100: + newTdSql.query('show dnodes') + res = newTdSql.queryResult + if newTdSql.checkEqual(res[1][4], "ready") == True: + break + + count+=1 sql = f"restore dnode 2" tdLog.info(sql) From e594c2bbd585f49261d8ab9c43e77e1973029d72 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Feb 2025 11:49:17 +0800 Subject: [PATCH 20/71] feat:[TD-33798]modify metadata logic for tmq --- source/common/src/tglobal.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 22 +- source/dnode/vnode/src/tq/tqUtil.c | 22 +- tests/parallel_test/cases.task | 1 - tests/parallel_test/cases_tdengine.task | 2 +- .../7-tmq/{tmq_ts-5473.py => tmq_c_test.py} | 8 + tests/system-test/7-tmq/tmq_taosx.py | 48 +-- tests/system-test/7-tmq/tmq_ts-5776.py | 39 --- utils/test/c/CMakeLists.txt | 8 + utils/test/c/tmq_td33798.c | 312 ++++++++++++++++++ 10 files changed, 375 insertions(+), 89 deletions(-) rename tests/system-test/7-tmq/{tmq_ts-5473.py => tmq_c_test.py} (81%) delete mode 100644 tests/system-test/7-tmq/tmq_ts-5776.py create mode 100644 utils/test/c/tmq_td33798.c diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2fd16b4f67..6369358008 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -188,7 +188,7 @@ char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/"; // tmq int32_t tmqMaxTopicNum = 20; -int32_t tmqRowSize = 4096; +int32_t tmqRowSize = 1000; // query int32_t tsQueryPolicy = 1; bool tsQueryTbNotExistAsEmpty = false; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 9d1b99f483..549a47d006 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -405,14 +405,16 @@ static void preProcessSubmitMsg(STqHandle* pHandle, const SMqPollReq* pRequest, } int64_t uid = pSubmitTbData->uid; - if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) { - tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid); - terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; - return; - } else { - int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES); - if (code != 0){ - tqError("failed to add table uid to hash, code:%d, uid:%"PRId64, code, uid); + if (pRequest->rawData) { + if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) { + tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + return; + } else { + int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES); + if (code != 0) { + tqError("failed to add table uid to hash, code:%d, uid:%" PRId64, code, uid); + } } } @@ -455,9 +457,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqData } code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver, rawList); TSDB_CHECK_CODE(code, lino, END); - if (pRequest->rawData) { - preProcessSubmitMsg(pHandle, pRequest, &rawList); - } + preProcessSubmitMsg(pHandle, pRequest, &rawList); // data could not contains same uid data in rawdata mode if (pRequest->rawData != 0 && terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){ goto END; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 4d908b5d57..7dc9a88164 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -266,7 +266,7 @@ END: return code; } -static int32_t buildBatchCreateContent(SMqDataRsp *taosxRsp, void** pBuf, int32_t *len){ +static int32_t buildCreateTbBatchReqBinary(SMqDataRsp *taosxRsp, void** pBuf, int32_t *len){ int32_t code = 0; SVCreateTbBatchReq pReq = {0}; pReq.nReqs = taosArrayGetSize(taosxRsp->createTableReq); @@ -280,7 +280,7 @@ static int32_t buildBatchCreateContent(SMqDataRsp *taosxRsp, void** pBuf, int32_ if (code < 0) { goto END; } - len += sizeof(SMsgHead); + *len += sizeof(SMsgHead); *pBuf = taosMemoryMalloc(*len); TQ_NULL_GO_TO_END(pBuf); SEncoder coder = {0}; @@ -337,7 +337,6 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, uint64_t st = taosGetTimestampMs(); int totalRows = 0; int32_t totalMetaRows = 0; - int32_t totalAutoMetaRows = 0; while (1) { int32_t savedEpoch = atomic_load_32(&pHandle->epoch); if (savedEpoch > pRequest->epoch) { @@ -348,7 +347,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) { - if (totalMetaRows > 0 || totalAutoMetaRows > 0) { + if (totalMetaRows > 0) { SEND_BATCH_META_RSP } SEND_DATA_RSP @@ -360,9 +359,6 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { - if (totalAutoMetaRows > 0){ - SEND_BATCH_META_RSP - } if (totalRows > 0) { SEND_DATA_RSP } @@ -402,7 +398,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, continue; } - if (totalMetaRows > 0) { + if (totalMetaRows > 0 && pHandle->fetchMeta != ONLY_META) { SEND_BATCH_META_RSP } @@ -415,10 +411,10 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest)); - if (pHandle->fetchMeta == ONLY_META){ + if (pHandle->fetchMeta == ONLY_META && taosArrayGetSize(taosxRsp.createTableReq) > 0){ int32_t len = 0; void *pBuf = NULL; - code = buildBatchCreateContent(&taosxRsp, &pBuf, &len); + code = buildCreateTbBatchReqBinary(&taosxRsp, &pBuf, &len); if (code == 0){ code = buildBatchMeta(&btMetaRsp, TDMT_VND_CREATE_TABLE, len, pBuf); } @@ -429,8 +425,10 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (code != 0){ continue; } - totalAutoMetaRows ++; - if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > pRequest->timeout)) { + totalMetaRows++; + if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || + (taosGetTimestampMs() - st > pRequest->timeout) || + (!pRequest->enableBatchMeta && !pRequest->useSnapshot)) { SEND_BATCH_META_RSP } continue; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0d14722aaf..b56dabe7cf 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -455,7 +455,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5776.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py diff --git a/tests/parallel_test/cases_tdengine.task b/tests/parallel_test/cases_tdengine.task index e52fe68957..18c50322ad 100644 --- a/tests/parallel_test/cases_tdengine.task +++ b/tests/parallel_test/cases_tdengine.task @@ -298,7 +298,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_c_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts4563.py diff --git a/tests/system-test/7-tmq/tmq_ts-5473.py b/tests/system-test/7-tmq/tmq_c_test.py similarity index 81% rename from tests/system-test/7-tmq/tmq_ts-5473.py rename to tests/system-test/7-tmq/tmq_c_test.py index ad08fa559c..a2ed4aa708 100644 --- a/tests/system-test/7-tmq/tmq_ts-5473.py +++ b/tests/system-test/7-tmq/tmq_c_test.py @@ -29,6 +29,14 @@ class TDTestCase: tdLog.info(cmdStr) os.system(cmdStr) + cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + cmdStr = '%s/build/bin/tmq_td33798'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + return def stop(self): diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 845e7229a8..053b29c013 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -677,34 +677,34 @@ class TDTestCase: tdSql.execute(f'drop database d1') def run(self): - self.consume_TS_5067_Test() - self.consumeTest() - self.consume_ts_4544() - self.consume_ts_4551() - self.consume_TS_4540_Test() - self.consume_td_31283() + # self.consume_TS_5067_Test() + # self.consumeTest() + # self.consume_ts_4544() + # self.consume_ts_4551() + # self.consume_TS_4540_Test() + # self.consume_td_31283() tdSql.prepare() self.checkWal1VgroupOnlyMeta() - self.checkWal1Vgroup() - self.checkSnapshot1Vgroup() - - self.checkWal1VgroupTable() - self.checkSnapshot1VgroupTable() - - self.checkWalMultiVgroups() - self.checkWalMultiVgroupsRawData() - self.checkSnapshotMultiVgroups() - - self.checkWalMultiVgroupsWithDropTable() - - self.checkSnapshotMultiVgroupsWithDropTable() - - self.checkSnapshot1VgroupBtmeta() - self.checkSnapshot1VgroupTableBtmeta() - self.checkSnapshotMultiVgroupsBtmeta() - self.checkSnapshotMultiVgroupsWithDropTableBtmeta() + # self.checkWal1Vgroup() + # self.checkSnapshot1Vgroup() + # + # self.checkWal1VgroupTable() + # self.checkSnapshot1VgroupTable() + # + # self.checkWalMultiVgroups() + # self.checkWalMultiVgroupsRawData() + # self.checkSnapshotMultiVgroups() + # + # self.checkWalMultiVgroupsWithDropTable() + # + # self.checkSnapshotMultiVgroupsWithDropTable() + # + # self.checkSnapshot1VgroupBtmeta() + # self.checkSnapshot1VgroupTableBtmeta() + # self.checkSnapshotMultiVgroupsBtmeta() + # self.checkSnapshotMultiVgroupsWithDropTableBtmeta() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmq_ts-5776.py b/tests/system-test/7-tmq/tmq_ts-5776.py deleted file mode 100644 index 738d69701f..0000000000 --- a/tests/system-test/7-tmq/tmq_ts-5776.py +++ /dev/null @@ -1,39 +0,0 @@ - -import taos -import sys -import time -import socket -import os -import threading - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * -from util.common import * -from taos.tmq import * -sys.path.append("./7-tmq") -from tmqCommon import * - -class TDTestCase: - updatecfgDict = {'debugFlag': 135, 'asynclog': 0} - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - #tdSql.init(conn.cursor(), logSql) # output sql.txt file - - def run(self): - buildPath = tdCom.getBuildPath() - cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath) - tdLog.info(cmdStr) - os.system(cmdStr) - - return - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index d73f91aad3..1b2716b8e5 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -8,6 +8,7 @@ add_executable(tmq_td32526 tmq_td32526.c) add_executable(tmq_td32187 tmq_td32187.c) add_executable(tmq_ts5776 tmq_ts5776.c) add_executable(tmq_td32471 tmq_td32471.c) +add_executable(tmq_td33798 tmq_td33798.c) add_executable(tmq_write_raw_test tmq_write_raw_test.c) add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) @@ -81,6 +82,13 @@ target_link_libraries( PUBLIC common PUBLIC os ) +target_link_libraries( + tmq_td33798 + PUBLIC ${TAOS_LIB} + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( tmq_td32526 PUBLIC ${TAOS_LIB} diff --git a/utils/test/c/tmq_td33798.c b/utils/test/c/tmq_td33798.c new file mode 100644 index 0000000000..a029b17b61 --- /dev/null +++ b/utils/test/c/tmq_td33798.c @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + +bool batchMeta = false; +int32_t consumeIndex = 0; +static TAOS* use_db() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +void checkBatchMeta(TAOS_RES* msg){ + char* result = tmq_get_json_meta(msg); + printf("meta result: %s\n", result); + switch (consumeIndex) { + case 0: + ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct11\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct10\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}]}") == 0); + break; + default: + ASSERT(0); + break; + } + + tmq_free_json_meta(result); +} + +void checkNonBatchMeta(TAOS_RES* msg){ + char* result = tmq_get_json_meta(msg); + printf("meta result: %s\n", result); + switch (consumeIndex) { + case 0: + ASSERT(strcmp(result, "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}") == 0); + break; + case 1: + ASSERT(strcmp(result, "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}") == 0); + break; + case 2: + ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}]}") == 0); + break; + case 3: + ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct11\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}]}") == 0); + break; + case 4: + ASSERT(strcmp(result, "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct10\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}") == 0); + break; + case 5: + ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}]}") == 0); + break; + default: + ASSERT(0); + break; + } + + tmq_free_json_meta(result); +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS* pConn = use_db(); + ASSERT (tmq_get_res_type(msg) == TMQ_RES_TABLE_META); + if (batchMeta){ + checkBatchMeta(msg); + } else { + checkNonBatchMeta(msg); + } + taos_close(pConn); +} + +int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct11 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct10 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taosSsleep(1); + pRes = taos_query(pConn, "insert into ct1 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a') ct2 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a') ct3 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_taosx vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists topic_db"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + buildDatabase(pConn, pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "create topic topic_db only meta as database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", batchMeta ? "batch" : "nonbatch"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); + if (batchMeta) { + tmq_conf_set(conf, "msg.enable.batchmeta", "1"); + } + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "topic_db"); + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + msg_process(tmqmessage); + consumeIndex++; + taos_free_result(tmqmessage); + } else { + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main(int argc, char* argv[]) { + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_list_t* topic_list = build_topic_list(); + tmq_t* tmq = build_consumer(); + basic_consume_loop(tmq, topic_list); + + batchMeta = true; + consumeIndex = 0; + tmq = build_consumer(); + basic_consume_loop(tmq, topic_list); + + tmq_list_destroy(topic_list); +} From d99709969bb47cc5f3f154f3c6c439436ca7f0b5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 21 Feb 2025 13:05:31 +0800 Subject: [PATCH 21/71] fix: add desc command --- tools/shell/src/shellAuto.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 87cdc546d2..f8ea42917c 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -112,6 +112,7 @@ SWords shellCommands[] = { {"create view as select", 0, 0, NULL}, {"compact database ", 0, 0, NULL}, #endif + {"desc ;", 0, 0, NULL}, {"describe ;", 0, 0, NULL}, {"delete from where ", 0, 0, NULL}, {"drop database ;", 0, 0, NULL}, From 512bb90e6a3f950b3bbf73791063404f1a928156 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 21 Feb 2025 14:49:20 +0800 Subject: [PATCH 22/71] fix: taosCli add three mode connection --- tests/army/cmdline/taosCli.py | 50 ++++++++++++++++++++++------------- tests/army/frame/caseBase.py | 6 +++++ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 35cceec933..b2bf62ecb4 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -53,31 +53,45 @@ class TDTestCase(TBase): tdSql.query(sql) tdSql.checkRows(2 + 1000) + + def checkResultWithMode(self, db, stb, arg): + result = "Query OK, 10 row(s)" + mode = arg[0] + rowh = arg[1] + rowv = arg[2] + idx = arg[3] + idxv = arg[4] + + # hori + cmd = f'{mode} -s "select * from {db}.{stb} limit 10' + rlist = self.taos(cmd + '"') + # line count + self.checkSame(len(rlist), rowh) + # last line + self.checkSame(rlist[idx][:len(result)], result) + + # vec + rlist = self.taos(cmd + '\G"') + # line count + self.checkSame(len(rlist), rowv) + self.checkSame(rlist[idxv], "*************************** 10.row ***************************") + # last line + self.checkSame(rlist[idx][:len(result)], result) + def checkBasic(self): tdLog.info(f"check describe show full.") # insert json = "cmdline/json/taosCli.json" db, stb, childCount, insertRows = self.insertBenchJson(json) - - result = "Query OK, 10 row(s)" - - # hori - cmd = f'-s "select * from {db}.{stb} limit 10' - rlist = etool.runBinFile("taos", cmd + '"') - # line count - self.checkSame(len(rlist), 18) - # last line - self.checkSame(rlist[-2][:len(result)], result) - - # vec - rlist = etool.runBinFile("taos", cmd + '\G"') - # line count - self.checkSame(len(rlist), 346) - self.checkSame(rlist[310], "*************************** 10.row ***************************") - # last line - self.checkSame(rlist[-2][:len(result)], result) + args = [ + ["", 18, 346, -2, 310], + ["-R", 22, 350, -3, 313], + ["-E http://localhost:6041", 21, 349, -3, 312] + ] + for arg in args: + self.checkResultWithMode(db, stb, arg) # run def run(self): diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 55fc5aa4c7..044ee88fec 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -19,6 +19,7 @@ import random import copy import json +import frame.etool import frame.eutil from frame.log import * from frame.sql import * @@ -278,6 +279,11 @@ class TBase: print(dics) return dics +# +# run bin file +# + def taos(self, command, show = True): + return frame.etool.runBinFile("taos", command, show) # # util From 6a81665ffab5ddf7e22a37b944eaf4279f667595 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 21 Feb 2025 14:51:40 +0800 Subject: [PATCH 23/71] fix/TS-6028-check-column-name-add-case --- source/dnode/mnode/impl/test/stb/stb.cpp | 89 ++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index e92231907f..438b4fd2e4 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -25,6 +25,7 @@ class MndTestStb : public ::testing::Test { void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); void* BuildDropDbReq(const char* dbname, int32_t* pContLen); void* BuildCreateStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen); void* BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen); void* BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen); void* BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname, @@ -137,6 +138,71 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { return pHead; } +void* MndTestStb::BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 2; + createReq.numOfTags = 4; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + strcpy(createReq.name, stbname); + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(field.name, "ts"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "col1"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 2; + field.type = TSDB_DATA_TYPE_TINYINT; + strcpy(field.name, "tag1"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_BIGINT; + strcpy(field.name, "tag2"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 16; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "tag3"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 16; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "tag3"); + taosArrayPush(createReq.pTags, &field); + } + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { SMAlterStbReq req = {0}; strcpy(req.name, stbname); @@ -896,3 +962,26 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { rpcFreeCont(pRsp->pCont); } } + +TEST_F(MndTestStb, 09_Create_Duplicate_Stb) { + const char* dbname = "1.d2"; + const char* stbname = "1.d2.stb"; + + { + int32_t contLen = 0; + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + rpcFreeCont(pRsp->pCont); + } + + { + int32_t contLen = 0; + void* pReq = BuildCreateStbDuplicateReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_TSC_DUP_COL_NAMES); + rpcFreeCont(pRsp->pCont); + } +} \ No newline at end of file From 2f05939f35a61f7f94b1357577d5a9a1d5768180 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 21 Feb 2025 16:03:51 +0800 Subject: [PATCH 24/71] fix: add version help and other command --- tests/army/cmdline/taosCli.py | 61 +++++++++++++++++++++++++++++++---- tests/army/frame/caseBase.py | 11 +++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index b2bf62ecb4..b8f8826b44 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -78,6 +78,13 @@ class TDTestCase(TBase): # last line self.checkSame(rlist[idx][:len(result)], result) + def checkDumpInOut(self, db, stb, insertRows): + self.taos(f'-s "select * from {db}.d0 >>d0.csv" ') + #self.taos(f'-s "delete from {db}.d0" ') + #self.taos(f'-s "insert into {db}.d0 file d0.out " ') + #sql = f"select count(*) from {db}.d0" + #tdSql.checkAgg(sql, insertRows) + def checkBasic(self): tdLog.info(f"check describe show full.") @@ -85,25 +92,67 @@ class TDTestCase(TBase): json = "cmdline/json/taosCli.json" db, stb, childCount, insertRows = self.insertBenchJson(json) + # native restful websock test args = [ ["", 18, 346, -2, 310], - ["-R", 22, 350, -3, 313], - ["-E http://localhost:6041", 21, 349, -3, 312] + ["-R -r", 22, 350, -3, 313], + ["-T 40 -E http://localhost:6041", 21, 349, -3, 312] ] for arg in args: self.checkResultWithMode(db, stb, arg) + # dump in/out + self.checkDumpInOut(db, stb, insertRows) + + + def checkVersion(self): + rlist1 = self.taos("-V") + rlist2 = self.taos("--version") + + self.checkSame(rlist1, rlist2) + self.checkSame(len(rlist1), 4) + + if len(rlist1[2]) < 42: + tdLog.exit("git commit id length is invalid: " + rlist1[2]) + + + def checkHelp(self): + # help + rlist1 = self.taos("--help") + rlist2 = self.taos("-?") + self.checkSame(rlist1, rlist2) + + # check return + strings = [ + "--auth=AUTH", + "--database=DATABASE", + "--version", + " --help" + ] + for string in strings: + self.checkListString(rlist1, string) + + def checkCommand(self): + self.taos(' -uroot -w 40 -ptaosdata -c /root/taos/ -s"show databases"') + # run def run(self): tdLog.debug(f"start to excute {__file__}") - # check basic - self.checkBasic() - # check show whole self.checkDescribe() - # full types show + # check basic + self.checkBasic() + + # version + self.checkVersion() + + # help + self.checkHelp() + + # check command + self.checkCommand() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 044ee88fec..725da440a9 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -326,6 +326,17 @@ class TBase: tdLog.exit(f"list is empty {tips}") + # check list have str + def checkListString(self, vlist, s): + for i in range(len(vlist)): + if vlist[i].find(s) != -1: + # found + tdLog.info(f"found {s} on index {i} , line={vlist[i]}") + return + + # not found + tdLog.exit(f"faild, not found {s} on list:{vlist}") + # # str util # From b6c455b2aaeb936dcc6f0a70c1dcf960010447c8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 21 Feb 2025 16:11:59 +0800 Subject: [PATCH 25/71] fix: need -B to run cases.task --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0d14722aaf..ed7efb13e3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -70,7 +70,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py ,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py -,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py +,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py -B ,,n,army,python3 ./test.py -f whole/checkErrorCode.py # From d3aedef555a7e9992c7d0e126d466efd82146842 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 21 Feb 2025 16:19:48 +0800 Subject: [PATCH 26/71] fix: -v return 5 lines check --- tests/army/cmdline/taosCli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index b8f8826b44..e8532730ff 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -110,7 +110,7 @@ class TDTestCase(TBase): rlist2 = self.taos("--version") self.checkSame(rlist1, rlist2) - self.checkSame(len(rlist1), 4) + self.checkSame(len(rlist1), 5) if len(rlist1[2]) < 42: tdLog.exit("git commit id length is invalid: " + rlist1[2]) From ff3ef5f5cab6a0f9bb89148876ddd73b469d5488 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 21 Feb 2025 16:49:21 +0800 Subject: [PATCH 27/71] fix/TD-33804-fix-ci-add-check-status --- tests/system-test/3-enterprise/restore/kill_restore_dnode.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/system-test/3-enterprise/restore/kill_restore_dnode.py b/tests/system-test/3-enterprise/restore/kill_restore_dnode.py index 4d0dca1952..478c0db2a6 100644 --- a/tests/system-test/3-enterprise/restore/kill_restore_dnode.py +++ b/tests/system-test/3-enterprise/restore/kill_restore_dnode.py @@ -78,10 +78,8 @@ class TDTestCase: count = 0 while count < 100: newTdSql.query('show dnodes') - res = newTdSql.queryResult - if newTdSql.checkEqual(res[1][4], "ready") == True: + if newTdSql.queryResult[1][4] == "ready": break - count+=1 sql = f"restore dnode 2" From fdc60a63369656b9d03a8918737c1f2838235c7c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 21 Feb 2025 18:20:26 +0800 Subject: [PATCH 28/71] fix:[TS-6028]check-column-name in sml --- source/client/src/clientSml.c | 47 +++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 376dea895c..c5ba653ad2 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -751,7 +751,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { } static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) { + SHashObj *schemaHashCheck, ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { @@ -759,15 +759,11 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SML_CHECK_NULL(kv); SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info)); - } - - for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); - SML_CHECK_NULL(kv); - if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { + if (taosHashGet(schemaHashCheck, kv->key, kv->keyLen) != NULL) { SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } + END: RETURN } @@ -998,14 +994,14 @@ static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTa END: RETURN } -static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, +static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ ESchemaAction action = SCHEMA_ACTION_NULL; SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; int32_t lino = 0; - SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, hashTmpCheck, &action, true)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); @@ -1029,14 +1025,14 @@ END: RETURN } -static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, +static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ ESchemaAction action = SCHEMA_ACTION_NULL; SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; int32_t lino = 0; - SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, hashTmpCheck, &action, false)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); @@ -1079,7 +1075,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } int32_t code = 0; int32_t lino = 0; - SHashObj *hashTmp = NULL; + SHashObj *colHashTmp = NULL; + SHashObj *tagHashTmp = NULL; STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; @@ -1119,17 +1116,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK); } - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - SML_CHECK_NULL(hashTmp); - SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); - SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); - taosHashClear(hashTmp); - SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); - SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); + colHashTmp = taosHashInit(pTableMeta->tableInfo.numOfColumns, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + tagHashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(colHashTmp); + SML_CHECK_NULL(tagHashTmp); + SML_CHECK_CODE(smlBuildTempHash(tagHashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); + SML_CHECK_CODE(smlBuildTempHash(colHashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); + + SML_CHECK_CODE(smlModifyTag(info, colHashTmp, tagHashTmp, &conn, sTableData, &pName, &pTableMeta)); + SML_CHECK_CODE(smlModifyCols(info, tagHashTmp, colHashTmp, &conn, sTableData, &pName, &pTableMeta)); needCheckMeta = true; - taosHashCleanup(hashTmp); - hashTmp = NULL; + taosHashCleanup(colHashTmp); + taosHashCleanup(tagHashTmp); + colHashTmp = NULL; + tagHashTmp = NULL; } else { uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code)); goto END; @@ -1153,7 +1154,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { END: taosHashCancelIterate(info->superTables, tmp); - taosHashCleanup(hashTmp); + taosHashCleanup(colHashTmp); + taosHashCleanup(tagHashTmp); taosMemoryFreeClear(pTableMeta); (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code, @@ -1924,3 +1926,4 @@ TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t * return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } + From b5e673f727a8b11374e594d6ffb4ac5bf3e3aad1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Feb 2025 19:05:18 +0800 Subject: [PATCH 29/71] feat:[TD-33798]modify metadata logic for tmq --- tests/parallel_test/cases.task | 1 - tests/system-test/7-tmq/tmq_taosx.py | 48 ++++++++++++++-------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6215d7d433..696693c051 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -470,7 +470,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 053b29c013..845e7229a8 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -677,34 +677,34 @@ class TDTestCase: tdSql.execute(f'drop database d1') def run(self): - # self.consume_TS_5067_Test() - # self.consumeTest() - # self.consume_ts_4544() - # self.consume_ts_4551() - # self.consume_TS_4540_Test() - # self.consume_td_31283() + self.consume_TS_5067_Test() + self.consumeTest() + self.consume_ts_4544() + self.consume_ts_4551() + self.consume_TS_4540_Test() + self.consume_td_31283() tdSql.prepare() self.checkWal1VgroupOnlyMeta() - # self.checkWal1Vgroup() - # self.checkSnapshot1Vgroup() - # - # self.checkWal1VgroupTable() - # self.checkSnapshot1VgroupTable() - # - # self.checkWalMultiVgroups() - # self.checkWalMultiVgroupsRawData() - # self.checkSnapshotMultiVgroups() - # - # self.checkWalMultiVgroupsWithDropTable() - # - # self.checkSnapshotMultiVgroupsWithDropTable() - # - # self.checkSnapshot1VgroupBtmeta() - # self.checkSnapshot1VgroupTableBtmeta() - # self.checkSnapshotMultiVgroupsBtmeta() - # self.checkSnapshotMultiVgroupsWithDropTableBtmeta() + self.checkWal1Vgroup() + self.checkSnapshot1Vgroup() + + self.checkWal1VgroupTable() + self.checkSnapshot1VgroupTable() + + self.checkWalMultiVgroups() + self.checkWalMultiVgroupsRawData() + self.checkSnapshotMultiVgroups() + + self.checkWalMultiVgroupsWithDropTable() + + self.checkSnapshotMultiVgroupsWithDropTable() + + self.checkSnapshot1VgroupBtmeta() + self.checkSnapshot1VgroupTableBtmeta() + self.checkSnapshotMultiVgroupsBtmeta() + self.checkSnapshotMultiVgroupsWithDropTableBtmeta() def stop(self): tdSql.close() From fe4f778aadc105a4a2274114986c24e6b382cfe1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 22 Feb 2025 18:03:12 +0800 Subject: [PATCH 30/71] fix: modify case taosCli.py --- tests/army/cmdline/taosCli.py | 77 +++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index e8532730ff..0a1ab02dba 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -78,12 +78,33 @@ class TDTestCase(TBase): # last line self.checkSame(rlist[idx][:len(result)], result) - def checkDumpInOut(self, db, stb, insertRows): + def checkDumpInOutMode(self, source, arg): + mode = arg[0] + self.taos(f'{mode} -s "source {source}" ') + + def checkDumpInOut(self): + args = [ + ["", 18, 346, -2, 310], + ["-R -r", 22, 350, -3, 313], + ["-T 40 -E http://localhost:6041", 21, 349, -3, 312] + ] + + source = "cmdline/data/source.sql" + + for arg in args: + self.checkDumpInOutMode(source, arg) + + db = "db" + insertRows = 5 self.taos(f'-s "select * from {db}.d0 >>d0.csv" ') - #self.taos(f'-s "delete from {db}.d0" ') - #self.taos(f'-s "insert into {db}.d0 file d0.out " ') - #sql = f"select count(*) from {db}.d0" - #tdSql.checkAgg(sql, insertRows) + self.taos(f'-s "delete from {db}.d0" ') + self.taos(f'-s "insert into {db}.d0 file d0.out " ') + sql = f"select count(*) from {db}.d0" + tdSql.checkAgg(sql, insertRows) + sql = f"select first(voltage) from {db}.d0" + tdSql.checkFirstValue(sql, 5) + sql = f"select last(voltage) from {db}.d0" + tdSql.checkFirstValue(sql, 5) def checkBasic(self): tdLog.info(f"check describe show full.") @@ -101,9 +122,6 @@ class TDTestCase(TBase): for arg in args: self.checkResultWithMode(db, stb, arg) - # dump in/out - self.checkDumpInOut(db, stb, insertRows) - def checkVersion(self): rlist1 = self.taos("-V") @@ -133,7 +151,46 @@ class TDTestCase(TBase): self.checkListString(rlist1, string) def checkCommand(self): - self.taos(' -uroot -w 40 -ptaosdata -c /root/taos/ -s"show databases"') + # check coredump + + # o logpath + char = 'a' + lname =f'-o "/root/log/{char * 1000}/" -s "quit;"' + queryOK = "Query OK" + + # invalid input check + args = [ + [lname, "failed to create log at"], + ['-uroot -w 40 -ptaosdata -c /root/taos/ -s"show databases"', queryOK], + ['-o "./current/log/files/" -s"show databases;"', queryOK], + ['-a ""', "Invalid auth"], + ['-s "quit;"', "Welcome to the TDengine Command Line Interface"], + ['-a "abc"', "[0x80000357]"], + ['-h "" -s "show dnodes;"', "Invalid host"], + ['-u "" -s "show dnodes;"', "Invalid user"], + ['-P "" -s "show dnodes;"', "Invalid port"], + ['-u "AA" -s "show dnodes;"', "failed to connect to server"], + ['-p"abc" -s "show dnodes;"', "[0x80000357]"], + ['-d "abc" -s "show dnodes;"', "[0x80000388]"], + ['-N 0 -s "show dnodes;"', "Invalid pktNum"], + ['-N 10 -s "show dnodes;"', queryOK], + ['-w 0 -s "show dnodes;"', "Invalid displayWidth"], + ['-w 10 -s "show dnodes;"', queryOK], + ['-W 10 -s "show dnodes;"', "taos: invalid option"], + ['-l 0 -s "show dnodes;"', "Invalid pktLen"], + ['-l 10 -s "show dnodes;"', queryOK], + ['-C', "buildinfo"], + ['-B -s "show dnodes;"', queryOK], + ['-s "help;"', "Timestamp expression Format"], + ['-s ""', "Invalid commands"], + ['-t', "2: service ok"] + ] + + for arg in args: + print(arg[0]) + print(arg[1]) + rlist = self.taos(arg[0]) + self.checkListString(rlist, arg[1]) # run def run(self): @@ -154,6 +211,8 @@ class TDTestCase(TBase): # check command self.checkCommand() + # check data in/out + self.checkDumpInOut() tdLog.success(f"{__file__} successfully executed") From d4c95697da72e8bfdf295fc0aaf6199a4f848e20 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 22 Feb 2025 18:28:12 +0800 Subject: [PATCH 31/71] fix: totalMemoryKB set 32G --- tests/army/cmdline/fullopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 7f97a6892a..f61a6437ca 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -30,7 +30,7 @@ class TDTestCase(TBase): updatecfgDict = { 'queryMaxConcurrentTables': '2K', 'streamMax': '1M', - 'totalMemoryKB': '1G', + 'totalMemoryKB': '32G', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" From c1d41e71f9adcd5c71f3f514a9c5288dbadf15ed Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 22 Feb 2025 19:02:52 +0800 Subject: [PATCH 32/71] fix: fullopts.py not passed --- tests/army/cmdline/fullopt.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index f61a6437ca..0cb1ee130c 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -30,7 +30,7 @@ class TDTestCase(TBase): updatecfgDict = { 'queryMaxConcurrentTables': '2K', 'streamMax': '1M', - 'totalMemoryKB': '32G', + 'totalMemoryKB': '32000000', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" @@ -47,10 +47,6 @@ class TDTestCase(TBase): # taosBenchmark run etool.benchMark(command = f"-d {self.db} -t {self.childtable_count} -n {self.insert_rows} -v 2 -y") - def checkQueryOK(self, rets): - if rets[-2][:9] != "Query OK,": - tdLog.exit(f"check taos -s return unexpect: {rets}") - def doTaos(self): tdLog.info(f"check taos command options...") @@ -71,10 +67,10 @@ class TDTestCase(TBase): ] # exec for option in options: - rets = etool.runBinFile("taos", f"-s \"alter local '{option}'\";") - self.checkQueryOK(rets) + rlist = self.taos(f"-s \"alter local '{option}'\"") + self.checkListString(rlist, "Query OK,") # error - etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\";") + etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\"") # help rets = etool.runBinFile("taos", "--help") From 338975b9b34a0663514ae2556c2e355e9547da2c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 22 Feb 2025 20:12:04 +0800 Subject: [PATCH 33/71] fix: taosCli.py passed --- tests/army/cmdline/taosCli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 0a1ab02dba..4619fe72fa 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -98,11 +98,11 @@ class TDTestCase(TBase): insertRows = 5 self.taos(f'-s "select * from {db}.d0 >>d0.csv" ') self.taos(f'-s "delete from {db}.d0" ') - self.taos(f'-s "insert into {db}.d0 file d0.out " ') + self.taos(f'-s "insert into {db}.d0 file d0.csv" ') sql = f"select count(*) from {db}.d0" tdSql.checkAgg(sql, insertRows) sql = f"select first(voltage) from {db}.d0" - tdSql.checkFirstValue(sql, 5) + tdSql.checkFirstValue(sql, 1) sql = f"select last(voltage) from {db}.d0" tdSql.checkFirstValue(sql, 5) @@ -176,7 +176,7 @@ class TDTestCase(TBase): ['-N 10 -s "show dnodes;"', queryOK], ['-w 0 -s "show dnodes;"', "Invalid displayWidth"], ['-w 10 -s "show dnodes;"', queryOK], - ['-W 10 -s "show dnodes;"', "taos: invalid option"], + ['-W 10 -s "show dnodes;"', None], ['-l 0 -s "show dnodes;"', "Invalid pktLen"], ['-l 10 -s "show dnodes;"', queryOK], ['-C', "buildinfo"], @@ -190,7 +190,8 @@ class TDTestCase(TBase): print(arg[0]) print(arg[1]) rlist = self.taos(arg[0]) - self.checkListString(rlist, arg[1]) + if arg[1] != None: + self.checkListString(rlist, arg[1]) # run def run(self): From c7e4680e3ae42a535f5f314ecbda721cbef6d8e8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 22 Feb 2025 22:03:48 +0800 Subject: [PATCH 34/71] fix: -R -h can coredump and memory leak --- tests/army/cmdline/taosCli.py | 72 +++++++++++++++++++++-------------- tools/shell/src/shellUtil.c | 30 ++++++++------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 4619fe72fa..897bb4e695 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -78,33 +78,12 @@ class TDTestCase(TBase): # last line self.checkSame(rlist[idx][:len(result)], result) - def checkDumpInOutMode(self, source, arg): - mode = arg[0] - self.taos(f'{mode} -s "source {source}" ') + # -B have some problem need todo + self.taos(f'{mode} -B -s "select * from {db}.{stb} where ts < 1"') - def checkDumpInOut(self): - args = [ - ["", 18, 346, -2, 310], - ["-R -r", 22, 350, -3, 313], - ["-T 40 -E http://localhost:6041", 21, 349, -3, 312] - ] - - source = "cmdline/data/source.sql" - - for arg in args: - self.checkDumpInOutMode(source, arg) - - db = "db" - insertRows = 5 - self.taos(f'-s "select * from {db}.d0 >>d0.csv" ') - self.taos(f'-s "delete from {db}.d0" ') - self.taos(f'-s "insert into {db}.d0 file d0.csv" ') - sql = f"select count(*) from {db}.d0" - tdSql.checkAgg(sql, insertRows) - sql = f"select first(voltage) from {db}.d0" - tdSql.checkFirstValue(sql, 1) - sql = f"select last(voltage) from {db}.d0" - tdSql.checkFirstValue(sql, 5) + # get empty result + rlist = self.taos(f'{mode} -s "select * from {db}.{stb} where ts < 1"') + self.checkListString(rlist, "Query OK, 0 row(s) in set") def checkBasic(self): tdLog.info(f"check describe show full.") @@ -123,6 +102,45 @@ class TDTestCase(TBase): self.checkResultWithMode(db, stb, arg) + def checkDumpInOutMode(self, source, arg, db, insertRows): + mode = arg[0] + self.taos(f'{mode} -s "source {source}" ') + self.taos(f'{mode} -s "select * from {db}.d0; >>d0.csv" ') + + # use db + rlist = self.taos(f'{mode} -s "use {db};" ') + self.checkListString(rlist, "Database changed") + # update sql + rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ') + self.checkListString(rlist, "Query O") + + # only native support csv import + if mode == "": + self.taos(f'{mode} -s "delete from {db}.d0" ') + self.taos(f'{mode} -s "insert into {db}.d0 file d0.csv" ') + + sql = f"select count(*) from {db}.d0" + self.taos(f'{mode} -B -s "{sql}" ') + tdSql.checkAgg(sql, insertRows) + sql = f"select first(voltage) from {db}.d0" + tdSql.checkFirstValue(sql, 1) + sql = f"select last(voltage) from {db}.d0" + tdSql.checkFirstValue(sql, 5) + + def checkDumpInOut(self): + args = [ + ["", 18], + ["-R ", 22], + ["-E http://localhost:6041", 21] + ] + + source = "cmdline/data/source.sql" + db = "db" + insertRows = 5 + for arg in args: + # insert + self.checkDumpInOutMode(source, arg, db, insertRows) + def checkVersion(self): rlist1 = self.taos("-V") rlist2 = self.taos("--version") @@ -187,8 +205,6 @@ class TDTestCase(TBase): ] for arg in args: - print(arg[0]) - print(arg[1]) rlist = self.taos(arg[0]) if arg[1] != None: self.checkListString(rlist, arg[1]) diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 0350d934a7..a8c1193ab8 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -122,6 +122,7 @@ void shellCheckServerStatus() { } while (1); } #ifdef WEBSOCKET +char dsn[1024] = "ws://localhost:6041"; void shellCheckConnectMode() { if (shell.args.dsn) { shell.args.cloud = true; @@ -144,21 +145,22 @@ void shellCheckConnectMode() { shell.args.restful = false; return; } - - if (shell.args.restful) { - if (!shell.args.host) { - shell.args.host = "localhost"; - } - if (!shell.args.port) { - shell.args.port = 6041; - } - shell.args.dsn = taosMemoryCalloc(1, 1024); - snprintf(shell.args.dsn, 1024, "ws://%s:%d", - shell.args.host, shell.args.port); - } - shell.args.cloud = false; - return; } + + if (shell.args.restful) { + if (!shell.args.host) { + shell.args.host = "localhost"; + } + if (!shell.args.port) { + shell.args.port = 6041; + } + shell.args.dsn = dsn; + snprintf(shell.args.dsn, 1024, "ws://%s:%d", + shell.args.host, shell.args.port); + } + shell.args.cloud = false; + return; + } #endif From 91b9a85cb0ab5af6385fbb612009be4794bd8210 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 22 Feb 2025 22:59:13 +0800 Subject: [PATCH 35/71] add nettest on fullopt.py --- tests/army/cmdline/fullopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 0cb1ee130c..4c320b87c6 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -154,6 +154,7 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taos", f'-n server', wait=False) time.sleep(3) + etool.exeBinFile("taos", f'-n client', wait=True) eos.exe("pkill -9 taos") # call enter password From f55252706e8916288730d6ddf2aae6e31f7b6325 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 10:07:37 +0800 Subject: [PATCH 36/71] fix: add check taos -n client result --- tests/army/cmdline/fullopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 4c320b87c6..500d481b5d 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -154,7 +154,8 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taos", f'-n server', wait=False) time.sleep(3) - etool.exeBinFile("taos", f'-n client', wait=True) + rlist = etool.exeBinFile("taos", f'-n client', wait=True) + self.checkListString(rlist, "total succ: 100/100") eos.exe("pkill -9 taos") # call enter password From f658affb81637c7fd909e17147ebaf5b71dd830e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 13:21:33 +0800 Subject: [PATCH 37/71] fix: fullopt.py passed --- tests/army/cmdline/fullopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 500d481b5d..4b99360676 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -154,7 +154,7 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taos", f'-n server', wait=False) time.sleep(3) - rlist = etool.exeBinFile("taos", f'-n client', wait=True) + rlist = self.taos("-n client") self.checkListString(rlist, "total succ: 100/100") eos.exe("pkill -9 taos") From d215cb40f3ebdb581c241510157685bd7c517338 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 13:30:31 +0800 Subject: [PATCH 38/71] fix: use db add many --- tests/army/cmdline/taosCli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 897bb4e695..b68db440fd 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -108,7 +108,7 @@ class TDTestCase(TBase): self.taos(f'{mode} -s "select * from {db}.d0; >>d0.csv" ') # use db - rlist = self.taos(f'{mode} -s "use {db};" ') + rlist = self.taos(f'{mode} -s "use {db};use {db};use {db};" ') self.checkListString(rlist, "Database changed") # update sql rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ') From 28d6f6b38bafc6a449a1d21f91084c30360a1c2a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 14:18:50 +0800 Subject: [PATCH 39/71] fix: use db; add show databases; --- tests/army/cmdline/taosCli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index b68db440fd..d3ac5c0d7b 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -108,7 +108,7 @@ class TDTestCase(TBase): self.taos(f'{mode} -s "select * from {db}.d0; >>d0.csv" ') # use db - rlist = self.taos(f'{mode} -s "use {db};use {db};use {db};" ') + rlist = self.taos(f'{mode} -s "show databases;use {db};use {db};use {db};quit;" ') self.checkListString(rlist, "Database changed") # update sql rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ') From cbf279470df807390b7633eed2e96e85d6dff449 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 16:13:33 +0800 Subject: [PATCH 40/71] fix: use db change output with printf --- tests/army/cmdline/taosCli.py | 5 +++-- tests/army/tools/taosdump/native/taosdumpCommandline.py | 3 +++ tools/shell/src/shellEngine.c | 3 +-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index d3ac5c0d7b..e1d1933b3f 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -108,7 +108,7 @@ class TDTestCase(TBase): self.taos(f'{mode} -s "select * from {db}.d0; >>d0.csv" ') # use db - rlist = self.taos(f'{mode} -s "show databases;use {db};use {db};use {db};quit;" ') + rlist = self.taos(f'{mode} -s "use {db};" ') self.checkListString(rlist, "Database changed") # update sql rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ') @@ -201,7 +201,8 @@ class TDTestCase(TBase): ['-B -s "show dnodes;"', queryOK], ['-s "help;"', "Timestamp expression Format"], ['-s ""', "Invalid commands"], - ['-t', "2: service ok"] + ['-t', "2: service ok"], + ['-uroot -p < cmdline/data/pwd.txt -s "show dnodes;"', queryOK], ] for arg in args: diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index 0e7c3dd29f..2779ad57d2 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -178,6 +178,9 @@ class TDTestCase(TBase): self.checkExcept(taosdump + " -A -o ") self.checkExcept(taosdump + " -A -o ./noexistpath/") self.checkExcept(taosdump + f" -d invalidAVRO -o {tmpdir}") + self.checkExcept(taosdump + f" -P invalidport") + self.checkExcept(taosdump + f" -D") + self.checkExcept(taosdump + f" -P 65536") # run def run(self): diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ebec4ca06c..a3e542a768 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -254,8 +254,7 @@ void shellRunSingleCommandImp(char *command) { } if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { - fprintf(stdout, "Database changed.\r\n\r\n"); - fflush(stdout); + printf("Database changed.\r\n\r\n"); // call back auto tab module callbackAutoTab(command, pSql, true); From 9b395d8b19d278bb1d498be3a70ee6a9f7f9bcc4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 16:42:27 +0800 Subject: [PATCH 41/71] fix: Remove TD_VER_COMPATIBLE define --- .../basic/json/{tmq.json => tmqBasic.json} | 0 .../{default_tmq_json.py => tmqBasic.py} | 2 +- tests/parallel_test/cases.task | 3 +- tools/taos-tools/inc/bench.h | 4 - tools/taos-tools/src/CMakeLists.txt | 29 +- tools/taos-tools/src/benchCommandOpt.c | 3 +- tools/taos-tools/src/benchJsonOpt.c | 2 - tools/taos-tools/src/benchSubscribe.c | 458 ------------------ tools/taos-tools/src/benchSys.c | 6 - tools/taos-tools/src/taosdump.c | 20 - 10 files changed, 9 insertions(+), 518 deletions(-) rename tests/army/tools/benchmark/basic/json/{tmq.json => tmqBasic.json} (100%) rename tests/army/tools/benchmark/basic/{default_tmq_json.py => tmqBasic.py} (96%) delete mode 100644 tools/taos-tools/src/benchSubscribe.c diff --git a/tests/army/tools/benchmark/basic/json/tmq.json b/tests/army/tools/benchmark/basic/json/tmqBasic.json similarity index 100% rename from tests/army/tools/benchmark/basic/json/tmq.json rename to tests/army/tools/benchmark/basic/json/tmqBasic.json diff --git a/tests/army/tools/benchmark/basic/default_tmq_json.py b/tests/army/tools/benchmark/basic/tmqBasic.py similarity index 96% rename from tests/army/tools/benchmark/basic/default_tmq_json.py rename to tests/army/tools/benchmark/basic/tmqBasic.py index e59b72d1f3..417c9670c0 100644 --- a/tests/army/tools/benchmark/basic/default_tmq_json.py +++ b/tests/army/tools/benchmark/basic/tmqBasic.py @@ -33,7 +33,7 @@ class TDTestCase(TBase): def run(self): tdSql.execute("drop topic if exists topic_0") binPath = etool.benchMarkFile() - cmd = "%s -f ./tools/benchmark/basic/json/default.json" % binPath + cmd = "%s -f ./tools/benchmark/basic/json/tmqBasic.json" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 2a5addc325..aca5a5a045 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -88,7 +88,6 @@ ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/custom_col_tag.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_tmq_json.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py @@ -140,6 +139,8 @@ ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestInsertWithJsonStmt-otherPara.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson.py -R ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/telnet_tcp.py -R + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmqBasic.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmq_case.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/cloud/cloud-test.py diff --git a/tools/taos-tools/inc/bench.h b/tools/taos-tools/inc/bench.h index 078bc1ac20..97c82ac2fa 100644 --- a/tools/taos-tools/inc/bench.h +++ b/tools/taos-tools/inc/bench.h @@ -617,10 +617,8 @@ typedef struct SDataBase_S { int durMinute; // duration minutes BArray *cfgs; BArray *superTbls; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 int32_t vgroups; BArray *vgArray; -#endif // TD_VER_COMPATIBLE_3_0_0_0 bool flush; } SDataBase; @@ -838,9 +836,7 @@ typedef struct SThreadInfo_S { BArray* delayList; uint64_t *query_delay_list; double avg_delay; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 SVGroup *vg; -#endif int posOfTblCreatingBatch; int posOfTblCreatingInterval; diff --git a/tools/taos-tools/src/CMakeLists.txt b/tools/taos-tools/src/CMakeLists.txt index da05133f89..708cd27f51 100644 --- a/tools/taos-tools/src/CMakeLists.txt +++ b/tools/taos-tools/src/CMakeLists.txt @@ -78,7 +78,6 @@ ADD_DEFINITIONS(-DBUILD_INFO="${BUILD_INFO}") MESSAGE(STATUS "build:${BUILD_INFO}") MESSAGE("") - # # build proj # @@ -88,15 +87,8 @@ INCLUDE_DIRECTORIES(/usr/local/taos/include) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/avro/lang/c/src) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/toolscJson/src) -if(NOT DEFINED TD_VER_COMPATIBLE) - SET(TD_VER_COMPATIBLE "3.0.0.0") -ENDIF() - -IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread) - INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex) - ADD_DEFINITIONS(-DTD_VER_COMPATIBLE_3_0_0_0) -ENDIF () +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread) +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex) MESSAGE("Current system is: ${CMAKE_SYSTEM_NAME}") @@ -128,21 +120,14 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEPENDENCIES(deps-snappy apache-avro) ADD_DEPENDENCIES(taosdump deps-jansson) ADD_DEPENDENCIES(taosdump deps-snappy) - IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ENDIF() + ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE () INCLUDE_DIRECTORIES(/usr/local/include) ADD_DEFINITIONS(-DDARWIN) LINK_DIRECTORIES(/usr/local/lib) SET(OS_ID "Darwin") - IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) + ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF() ENDIF () @@ -337,11 +322,7 @@ ELSE () SET(CMAKE_C_STANDARD 11) SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8") SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8") - IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) - ELSE () - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ENDIF () + ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) ADD_EXECUTABLE(taosdump taosdump.c dumpUtil.c wsdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c) ADD_DEPENDENCIES(apache-avro tools-zlib) diff --git a/tools/taos-tools/src/benchCommandOpt.c b/tools/taos-tools/src/benchCommandOpt.c index 8c18b54eac..9bc57bff01 100644 --- a/tools/taos-tools/src/benchCommandOpt.c +++ b/tools/taos-tools/src/benchCommandOpt.c @@ -245,9 +245,8 @@ void initArgument() { g_arguments->trying_interval = 0; g_arguments->iface = TAOSC_IFACE; g_arguments->rest_server_ver_major = -1; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 g_arguments->inputted_vgroups = -1; -#endif + g_arguments->mistMode = false; initDatabase(); diff --git a/tools/taos-tools/src/benchJsonOpt.c b/tools/taos-tools/src/benchJsonOpt.c index 3e41908668..4791385741 100644 --- a/tools/taos-tools/src/benchJsonOpt.c +++ b/tools/taos-tools/src/benchJsonOpt.c @@ -2254,7 +2254,6 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { return 0; } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 static int getMetaFromTmqJsonFile(tools_cJSON *json) { int32_t code = -1; @@ -2422,7 +2421,6 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { TMQ_PARSE_OVER: return code; } -#endif int readJsonConfig(char * file) { int32_t code = -1; diff --git a/tools/taos-tools/src/benchSubscribe.c b/tools/taos-tools/src/benchSubscribe.c deleted file mode 100644 index 7f0d7e1579..0000000000 --- a/tools/taos-tools/src/benchSubscribe.c +++ /dev/null @@ -1,458 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the MIT license as published by the Free Software - * Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - */ - -#include -#include "benchLog.h" - -extern int g_majorVersionOfClient; - -static void stable_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param, - int code) { - if (res == NULL || taos_errno(res) != 0) { - errorPrint("failed to subscribe result, code:%d, reason:%s\n", - code, taos_errstr(res)); - return; - } - - if (param) fetchResult(res, ((threadInfo *)param)->filePath); - // tao_unsubscribe() will free result. -} - -static void specified_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param, - int code) { - if (res == NULL || taos_errno(res) != 0) { - errorPrint("failed to subscribe result, code:%d, reason:%s\n", - code, taos_errstr(res)); - return; - } - - if (param) fetchResult(res, ((threadInfo *)param)->filePath); - // tao_unsubscribe() will free result. -} - -static TAOS_SUB *subscribeImpl(QUERY_CLASS class, threadInfo *pThreadInfo, - char *sql, char *topic, bool restart, - uint64_t interval) { - TAOS_SUB *tsub = NULL; - - if (taos_select_db(pThreadInfo->conn->taos, g_queryInfo.dbName)) { - errorPrint("failed to select database(%s)\n", g_queryInfo.dbName); - return NULL; - } - if ((SPECIFIED_CLASS == class) && - (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { - tsub = taos_subscribe( - pThreadInfo->conn->taos, restart, - topic, sql, specified_sub_callback, - (void *)pThreadInfo, - (int)g_queryInfo.specifiedQueryInfo.subscribeInterval); - } else if ((STABLE_CLASS == class) && - (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { - tsub = taos_subscribe(pThreadInfo->conn->taos, restart, topic, sql, - stable_sub_callback, (void *)pThreadInfo, - (int)g_queryInfo.superQueryInfo.subscribeInterval); - } else { - tsub = taos_subscribe(pThreadInfo->conn->taos, - restart, topic, sql, NULL, - NULL, (int)interval); - } - - if (tsub == NULL) { - errorPrint("failed to create subscription. topic:%s, sql:%s\n", - topic, sql); - return NULL; - } - - return tsub; -} - -static void *specifiedSubscribe(void *sarg) { - int32_t *code = benchCalloc(1, sizeof(int32_t), false); - *code = -1; - threadInfo *pThreadInfo = (threadInfo *)sarg; -#ifdef LINUX - prctl(PR_SET_NAME, "specSub"); -#endif - sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - "taosbenchmark-subscribe-%" PRIu64 "-%d", pThreadInfo->querySeq, - pThreadInfo->threadID); - SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, - pThreadInfo->querySeq); - if (sql->result[0] != - '\0') { - snprintf(pThreadInfo->filePath, MAX_PATH_LEN, - "%s-%d", sql->result, pThreadInfo->threadID); - } - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, sql->command, - g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeRestart, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { - goto free_of_specified_subscribe; - } - - // start loop to consume result - - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; - while ((g_queryInfo.specifiedQueryInfo - .endAfterConsume[pThreadInfo->querySeq] == -1) || - (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] < - g_queryInfo.specifiedQueryInfo - .endAfterConsume[pThreadInfo->querySeq])) { - infoPrint( - "consumed[%d]: %d, endAfterConsum[%" PRId64 "]: %d\n", - pThreadInfo->threadID, - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID], - pThreadInfo->querySeq, - g_queryInfo.specifiedQueryInfo - .endAfterConsume[pThreadInfo->querySeq]); - - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = - taos_consume( - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); - if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { - if (sql->result[0] != 0) { - snprintf(pThreadInfo->filePath, MAX_PATH_LEN, - "%s-%d", sql->result, pThreadInfo->threadID); - } - fetchResult( - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo->filePath); - - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID]++; - if ((g_queryInfo.specifiedQueryInfo - .resubAfterConsume[pThreadInfo->querySeq] != -1) && - (g_queryInfo.specifiedQueryInfo - .consumed[pThreadInfo->threadID] >= - g_queryInfo.specifiedQueryInfo - .resubAfterConsume[pThreadInfo->querySeq])) { - infoPrint( - "keepProgress:%d, resub specified query: %" PRIu64 - "\n", - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, - pThreadInfo->querySeq); - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = - 0; - taos_unsubscribe( - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - g_queryInfo.specifiedQueryInfo - .tsub[pThreadInfo->threadID] = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, sql->command, - g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeRestart, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == g_queryInfo.specifiedQueryInfo - .tsub[pThreadInfo->threadID]) { - goto free_of_specified_subscribe; - } - } - } - } - *code = 0; - taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); -free_of_specified_subscribe: - return code; -} - -static void *superSubscribe(void *sarg) { - int32_t *code = benchCalloc(1, sizeof(int32_t), false); - *code = -1; - threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB * tsub[MAX_QUERY_SQL_COUNT] = {0}; - uint64_t tsubSeq; - char * subSqlStr = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); -#ifdef LINUX - prctl(PR_SET_NAME, "superSub"); -#endif - if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint( - "The table number(%" PRId64 - ") of the thread is more than max query sql count: %d\n", - pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); - goto free_of_super_subscribe; - } - - char topic[32] = {0}; - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - tsubSeq = i - pThreadInfo->start_table_from; - sprintf(topic, "taosbenchmark-subscribe-%" PRIu64 "-%" PRIu64 "", i, - pThreadInfo->querySeq); - memset(subSqlStr, 0, TSDB_MAX_ALLOWED_SQL_LEN); - replaceChildTblName( - g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlStr, - (int)i); - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - snprintf(pThreadInfo->filePath, MAX_PATH_LEN, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - tsub[tsubSeq] = - subscribeImpl(STABLE_CLASS, pThreadInfo, subSqlStr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - g_queryInfo.superQueryInfo.subscribeInterval); - if (NULL == tsub[tsubSeq]) { - goto free_of_super_subscribe; - } - } - - // start loop to consume result - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) { - consumed[i] = 0; - } - TAOS_RES *res = NULL; - - uint64_t st = 0, et = 0; - - while ( - (g_queryInfo.superQueryInfo.endAfterConsume == -1) || - (g_queryInfo.superQueryInfo.endAfterConsume > - consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) { - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - tsubSeq = i - pThreadInfo->start_table_from; - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } - - st = toolsGetTimestampMs(); - perfPrint( - "st: %" PRIu64 " et: %" PRIu64 " st-et: %" PRIu64 "\n", - st, et, (st - et)); - res = taos_consume(tsub[tsubSeq]); - et = toolsGetTimestampMs(); - perfPrint( - "st: %" PRIu64 " et: %" PRIu64 " delta: %" PRIu64 "\n", - st, et, (et - st)); - - if (res) { - if (g_queryInfo.superQueryInfo - .result[pThreadInfo->querySeq][0] != 0) { - snprintf(pThreadInfo->filePath, - MAX_PATH_LEN, - "%s-%d", - g_queryInfo.superQueryInfo - .result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - fetchResult(res, pThreadInfo->filePath); - consumed[tsubSeq]++; - - if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1) && - (consumed[tsubSeq] >= - g_queryInfo.superQueryInfo.resubAfterConsume)) { - taos_unsubscribe( - tsub[tsubSeq], - g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[tsubSeq] = 0; - tsub[tsubSeq] = subscribeImpl( - STABLE_CLASS, pThreadInfo, subSqlStr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - g_queryInfo.superQueryInfo.subscribeInterval); - if (NULL == tsub[tsubSeq]) { - goto free_of_super_subscribe; - } - } - } - } - } - taos_free_result(res); - - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - tsubSeq = i - pThreadInfo->start_table_from; - taos_unsubscribe(tsub[tsubSeq], 0); - } - *code = 0; -free_of_super_subscribe: - - tmfree(subSqlStr); - return code; -} - -int subscribeTestProcess() { - prompt(0); - - if (REST_IFACE == g_queryInfo.iface) { - encodeAuthBase64(); - } - if (0 != g_queryInfo.superQueryInfo.sqlCount) { - SBenchConn* conn = initBenchConn(); - if (conn == NULL) { - return -1; - } - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - if (3 == g_majorVersionOfClient) { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", - g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); - } else { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(TBNAME) FROM %s.%s", - g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); - } - TAOS_RES *res = taos_query(conn->taos, cmd); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(res); - TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { - if (0 == strlen((char *)(row[0]))) { - errorPrint("stable %s have no child table\n", - g_queryInfo.superQueryInfo.stbName); - return -1; - } - char temp[256] = {0}; - taos_print_row(temp, row, fields, num_fields); - g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp); - } - infoPrint("%s's childTblCount: %" PRId64 "\n", - g_queryInfo.superQueryInfo.stbName, - g_queryInfo.superQueryInfo.childTblCount); - taos_free_result(res); - g_queryInfo.superQueryInfo.childTblName = - benchCalloc(g_queryInfo.superQueryInfo.childTblCount, - sizeof(char *), false); - if (getAllChildNameOfSuperTable( - conn->taos, g_queryInfo.dbName, - g_queryInfo.superQueryInfo.stbName, - g_queryInfo.superQueryInfo.childTblName, - g_queryInfo.superQueryInfo.childTblCount)) { - closeBenchConn(conn); - return -1; - } - closeBenchConn(conn); - } - - pthread_t * pids = NULL; - threadInfo *infos = NULL; - - pthread_t * pidsOfStable = NULL; - threadInfo *infosOfStable = NULL; - - //==== create threads for query for specified table - int sqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; - if (sqlCount > 0) { - pids = benchCalloc(1, sqlCount - *g_queryInfo.specifiedQueryInfo.concurrent - *sizeof(pthread_t), false); - infos = benchCalloc(1, sqlCount - *g_queryInfo.specifiedQueryInfo.concurrent - *sizeof(threadInfo), false); - for (int i = 0; i < sqlCount; i++) { - for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; - j++) { - uint64_t seq = - i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *pThreadInfo = infos + seq; - pThreadInfo->threadID = (int)seq; - pThreadInfo->querySeq = i; - pThreadInfo->conn = initBenchConn(); - pthread_create(pids + seq, NULL, specifiedSubscribe, - pThreadInfo); - } - } - - for (int i = 0; i < sqlCount; i++) { - for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; - j++) { - uint64_t seq = - i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *pThreadInfo = infos + seq; - void *result; - pthread_join(pids[seq], &result); - if (*(int32_t *)result) { - g_fail = true; - } - closeBenchConn(pThreadInfo->conn); - tmfree(result); - } - } - } - - //==== create threads for super table query - if (g_queryInfo.superQueryInfo.sqlCount > 0 && - g_queryInfo.superQueryInfo.threadCnt > 0) { - pidsOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount * - g_queryInfo.superQueryInfo.threadCnt * - sizeof(pthread_t), false); - - infosOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount * - g_queryInfo.superQueryInfo.threadCnt * - sizeof(threadInfo), false); - - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } - - for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - uint64_t tableFrom = 0; - for (int j = 0; j < threads; j++) { - uint64_t seq = i * threads + j; - threadInfo *pThreadInfo = infosOfStable + seq; - pThreadInfo->threadID = (int)seq; - pThreadInfo->querySeq = i; - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = j < b ? a + 1 : a; - pThreadInfo->end_table_to = - j < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->conn = initBenchConn(); - pthread_create(pidsOfStable + seq, NULL, superSubscribe, - pThreadInfo); - } - } - - g_queryInfo.superQueryInfo.threadCnt = threads; - - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - for (int j = 0; j < threads; j++) { - uint64_t seq = (uint64_t)i * threads + j; - threadInfo *pThreadInfo = infosOfStable + seq; - void * result; - pthread_join(pidsOfStable[seq], &result); - if (*(int32_t *)result) { - g_fail = true; - } - closeBenchConn(pThreadInfo->conn); - tmfree(result); - } - } - } - - tmfree((char *)pids); - tmfree((char *)infos); - - tmfree((char *)pidsOfStable); - tmfree((char *)infosOfStable); - // - if (g_fail) { - return -1; - } - return 0; -} diff --git a/tools/taos-tools/src/benchSys.c b/tools/taos-tools/src/benchSys.c index 9e29701515..d7e44c045f 100644 --- a/tools/taos-tools/src/benchSys.c +++ b/tools/taos-tools/src/benchSys.c @@ -77,9 +77,7 @@ void benchPrintHelp() { printf("%s%s%s%s\r\n", indent, "-W,", indent, BENCH_DSN); printf("%s%s%s%s\r\n", indent, "-D,", indent, BENCH_TIMEOUT); #endif -#ifdef TD_VER_COMPATIBLE_3_0_0_0 printf("%s%s%s%s\r\n", indent, "-v,", indent, BENCH_VGROUPS); -#endif printf("%s%s%s%s\r\n", indent, "-V,", indent, BENCH_VERSION); printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL); } @@ -199,9 +197,7 @@ static struct argp_option bench_options[] = { #endif {"keep-trying", 'k', "NUMBER", 0, BENCH_KEEPTRYING}, {"trying-interval", 'z', "NUMBER", 0, BENCH_TRYING_INTERVAL}, -#ifdef TD_VER_COMPATIBLE_3_0_0_0 {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, -#endif {"version", 'V', 0, 0, BENCH_VERSION}, {"nodrop", 'Q', 0, 0, BENCH_NODROP}, {0} @@ -642,14 +638,12 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { g_arguments->timeout = atoi(arg); break; #endif -#ifdef TD_VER_COMPATIBLE_3_0_0_0 case 'v': if (!toolsIsStringNumber(arg)) { errorPrintReqArg2(CUS_PROMPT"Benchmark", "v"); } g_arguments->inputted_vgroups = atoi(arg); break; -#endif case 'Q': database->drop = false; g_argFlag |= ARG_OPT_NODROP; diff --git a/tools/taos-tools/src/taosdump.c b/tools/taos-tools/src/taosdump.c index d27812b67a..35fae99c6a 100644 --- a/tools/taos-tools/src/taosdump.c +++ b/tools/taos-tools/src/taosdump.c @@ -5831,26 +5831,6 @@ static int64_t dumpInAvroDataImpl( #endif } } // tbName -#ifndef TD_VER_COMPATIBLE_3_0_0_0 - else { - // 2.6 need call taos_stmt_set_tbname every loop - const int escapedTbNameLen = TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 3; - char *escapedTbName = calloc(1, escapedTbNameLen); - snprintf(escapedTbName, escapedTbNameLen, "%s%s%s", - g_escapeChar, tbName, g_escapeChar); - - if (0 != taos_stmt_set_tbname(stmt, escapedTbName)) { - errorPrint("Failed to execute taos_stmt_set_tbname(%s)." - "reason: %s\n", - escapedTbName, taos_stmt_errstr(stmt)); - free(escapedTbName); - freeTbNameIfLooseMode(tbName); - tbName = NULL; - continue; - } - free(escapedTbName); - } -#endif debugPrint("%s() LN%d, count: %"PRId64"\n", __func__, __LINE__, count); From 0c2695ec6d693456aa2aa596a8b25968924927d3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 18:24:55 +0800 Subject: [PATCH 42/71] tmq add tmpBasic.py --- tests/army/frame/caseBase.py | 22 ++++-- tests/army/frame/eos.py | 22 +++--- tests/army/frame/etool.py | 6 +- .../benchmark/basic/json/tmqBasicInsert.json | 71 +++++++++++++++++++ .../benchmark/basic/json/tmqBasicPara.json | 29 ++++++++ .../json/{tmqBasic.json => tmqBasicSequ.json} | 13 ++-- tests/army/tools/benchmark/basic/tmqBasic.py | 37 +++------- 7 files changed, 153 insertions(+), 47 deletions(-) create mode 100644 tests/army/tools/benchmark/basic/json/tmqBasicInsert.json create mode 100644 tests/army/tools/benchmark/basic/json/tmqBasicPara.json rename tests/army/tools/benchmark/basic/json/{tmqBasic.json => tmqBasicSequ.json} (67%) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 725da440a9..649bc231b0 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -331,11 +331,11 @@ class TBase: for i in range(len(vlist)): if vlist[i].find(s) != -1: # found - tdLog.info(f"found {s} on index {i} , line={vlist[i]}") + tdLog.info(f'found "{s}" on index {i} , line={vlist[i]}') return # not found - tdLog.exit(f"faild, not found {s} on list:{vlist}") + tdLog.exit(f'faild, not found "{s}" on list:{vlist}') # # str util @@ -353,7 +353,7 @@ class TBase: # taosBenchmark # - # run taosBenchmark and check insert Result + # insert def insertBenchJson(self, jsonFile, options="", checkStep=False): # exe insert cmd = f"{options} -f {jsonFile}" @@ -420,4 +420,18 @@ class TBase: if vgroups != None: tdSql.checkData(0, 0, vgroups) - return db, stb,child_count, insert_rows \ No newline at end of file + return db, stb,child_count, insert_rows + + + # tmq + def tmqBenchJson(self, jsonFile, options="", checkStep=False): + # exe insert + command = f"{options} -f {jsonFile}" + rlist = frame.etool.runBinFile("taosBenchmark", command, checkRun = True) + + # + # check insert result + # + print(rlist) + + return rlist \ No newline at end of file diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 363714a6d1..9e502c3e59 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -52,8 +52,11 @@ def isArm64Cpu(): # # wait util execute file finished -def exe(file): - return os.system(file) +def exe(command, show = False): + code = os.system(command) + if show: + print(f"eos.exe retcode={code} command:{command}") + return code # execute file and return immediately def exeNoWait(file): @@ -64,13 +67,13 @@ def exeNoWait(file): return exe(cmd) # run return output and error -def run(command, show=True): +def run(command, show = True): # out to file id = time.clock_gettime_ns(time.CLOCK_REALTIME) % 100000 out = f"out_{id}.txt" err = f"err_{id}.txt" - ret = exe(command + f" 1>{out} 2>{err}") + code = exe(command + f" 1>{out} 2>{err}", show) # read from file output = readFileContext(out) @@ -80,14 +83,17 @@ def run(command, show=True): if os.path.exists(out): os.remove(out) if os.path.exists(err): - os.remove(err) + os.remove(err) - return output, error + return output, error, code # return list after run -def runRetList(command, timeout=10): - output,error = run(command, timeout) +def runRetList(command, show = True, checkRun = False): + output, error, code = run(command, show) + if checkRun and code != 0: + print(f"eos.runRetList checkRun return code failed. code={code} error={error}") + assert code == 0 return output.splitlines() # diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index 9065a3dede..164e5e127d 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -51,7 +51,7 @@ def benchMark(command = "", json = "") : # run if command != "": - frame.eos.exe(bmFile + " " + command) + status = frame.eos.run(bmFile + " " + command) if json != "": cmd = f"{bmFile} -f {json}" print(cmd) @@ -66,7 +66,7 @@ def curFile(fullPath, filename): # run build/bin file -def runBinFile(fname, command, show=True): +def runBinFile(fname, command, show = True, checkRun = False): binFile = frame.epath.binFile(fname) if frame.eos.isWin(): binFile += ".exe" @@ -74,7 +74,7 @@ def runBinFile(fname, command, show=True): cmd = f"{binFile} {command}" if show: tdLog.info(cmd) - return frame.eos.runRetList(cmd) + return frame.eos.runRetList(cmd, show, checkRun) # exe build/bin file def exeBinFile(fname, command, wait=True, show=True): diff --git a/tests/army/tools/benchmark/basic/json/tmqBasicInsert.json b/tests/army/tools/benchmark/basic/json/tmqBasicInsert.json new file mode 100644 index 0000000000..62ab67de2f --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/tmqBasicInsert.json @@ -0,0 +1,71 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count": 2, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes", + "vgroups": 4 + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 10, + "insert_rows": 100, + "timestamp_step": 1, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} + ], + "tags":[ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} + ] + } + ] + } + ] +} diff --git a/tests/army/tools/benchmark/basic/json/tmqBasicPara.json b/tests/army/tools/benchmark/basic/json/tmqBasicPara.json new file mode 100644 index 0000000000..c5b9c5b4f2 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/tmqBasicPara.json @@ -0,0 +1,29 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "confirm_parameter_prompt": "no", + "result_file": "./tmq_result_para.txt", + "tmq_info": { + "concurrent": 2, + "poll_delay": 3000, + "create_mode": "parallel", + "group_mode": "independent", + "client.id": "clientId", + "auto.offset.reset": "earliest", + "enable.auto.commit": "true", + "auto.commit.interval.ms": 1000, + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "false", + "rows_file": "./rows_file_para", + "expect_rows": 100, + "topic_list": [ + {"name": "topic_benchmark_d0", "sql": "select * from test.d0;"} + ] + } +} diff --git a/tests/army/tools/benchmark/basic/json/tmqBasic.json b/tests/army/tools/benchmark/basic/json/tmqBasicSequ.json similarity index 67% rename from tests/army/tools/benchmark/basic/json/tmqBasic.json rename to tests/army/tools/benchmark/basic/json/tmqBasicSequ.json index cf0eb1b297..d508fe71eb 100644 --- a/tests/army/tools/benchmark/basic/json/tmqBasic.json +++ b/tests/army/tools/benchmark/basic/json/tmqBasicSequ.json @@ -5,12 +5,13 @@ "port": 6030, "user": "root", "password": "taosdata", - "databases": "db", + "databases": "test", "confirm_parameter_prompt": "no", - "result_file": "./tmq_result1.txt", + "result_file": "./tmq_result_sequ.txt", "tmq_info": { - "concurrent": 2, + "concurrent": 4, "poll_delay": 3000, + "group_mode": "independent", "group.id": "grpId_0", "client.id": "clientId", "auto.offset.reset": "earliest", @@ -19,10 +20,10 @@ "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "false", - "rows_file": "./consumed_rows1", - "expect_rows": 50, + "rows_file": "./rows_file_sequ", + "expect_rows": 1000, "topic_list": [ - {"name": "tmq_topic_0", "sql": "select c0 from db.stb;"} + {"name": "topic_benchmark_meters", "sql": "select * from test.meters;"} ] } } diff --git a/tests/army/tools/benchmark/basic/tmqBasic.py b/tests/army/tools/benchmark/basic/tmqBasic.py index 417c9670c0..66d2290a1f 100644 --- a/tests/army/tools/benchmark/basic/tmqBasic.py +++ b/tests/army/tools/benchmark/basic/tmqBasic.py @@ -25,37 +25,22 @@ from frame import * class TDTestCase(TBase): def caseDescription(self): """ - [TD-11510] taosBenchmark test cases + taosBenchmark tmp->Basic test cases """ - - def run(self): - tdSql.execute("drop topic if exists topic_0") - binPath = etool.benchMarkFile() - cmd = "%s -f ./tools/benchmark/basic/json/tmqBasic.json" % binPath - tdLog.info("%s" % cmd) - os.system("%s" % cmd) - tdSql.execute("reset query cache") + + # insert data + json = "tools/benchmark/basic/json/tmqBasicInsert.json" + db, stb, child_count, insert_rows = self.insertBenchJson(json, checkStep = True) - tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000") + # tmq Sequ + json = "tools/benchmark/basic/json/tmqBasicSequ.json" + self.tmqBenchJson(json) - cmd = "%s -f ./tools/benchmark/basic/json/tmq.json " % binPath - tdLog.info("%s" % cmd) - os.system("%s" % cmd) - sleep(15) - - # try: - # for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"): - # fields = line.split() - - # pid = fields[0] - - # os.kill(int(pid), signal.SIGINT) - # time.sleep(3) - # print("taosBenchmark be killed on purpose") - # except: - # tdLog.exit("failed to kill taosBenchmark") + # tmq Parallel + json = "tools/benchmark/basic/json/tmqBasicPara.json" + self.tmqBenchJson(json) def stop(self): tdSql.close() From 462cf902789e4ffd42a365eff890da9c7a3f31c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 18:30:09 +0800 Subject: [PATCH 43/71] fix: add pwd.txt for taos -p check input password --- tests/army/cmdline/data/pwd.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/army/cmdline/data/pwd.txt diff --git a/tests/army/cmdline/data/pwd.txt b/tests/army/cmdline/data/pwd.txt new file mode 100644 index 0000000000..771ffcffe6 --- /dev/null +++ b/tests/army/cmdline/data/pwd.txt @@ -0,0 +1 @@ +taosdata \ No newline at end of file From e4a3159118060939d2d2871cf455c908eae5a8fd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 23 Feb 2025 19:08:20 +0800 Subject: [PATCH 44/71] CMakeFile less del ENDIF --- tools/taos-tools/src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/taos-tools/src/CMakeLists.txt b/tools/taos-tools/src/CMakeLists.txt index 708cd27f51..1f0899db5c 100644 --- a/tools/taos-tools/src/CMakeLists.txt +++ b/tools/taos-tools/src/CMakeLists.txt @@ -128,7 +128,6 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(OS_ID "Darwin") ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ENDIF() ENDIF () # websocket From 237c9f2b28322545b26fcb5f2817a9c8f4a7b4ca Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 24 Feb 2025 15:38:28 +0800 Subject: [PATCH 45/71] fix: use db; report error --- tests/army/cmdline/taosCli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index e1d1933b3f..f58d727d2b 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -62,6 +62,10 @@ class TDTestCase(TBase): idx = arg[3] idxv = arg[4] + # use db + rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') + self.checkListString(rlist, "Database changed") + # hori cmd = f'{mode} -s "select * from {db}.{stb} limit 10' rlist = self.taos(cmd + '"') @@ -108,8 +112,7 @@ class TDTestCase(TBase): self.taos(f'{mode} -s "select * from {db}.d0; >>d0.csv" ') # use db - rlist = self.taos(f'{mode} -s "use {db};" ') - self.checkListString(rlist, "Database changed") + rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') # update sql rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ') self.checkListString(rlist, "Query O") From 471ec0a5d6137cf2715469ecad3eff603148d343 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 24 Feb 2025 15:55:54 +0800 Subject: [PATCH 46/71] fix: -R no need do use db --- tests/army/cmdline/taosCli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index f58d727d2b..8e2c05c166 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -63,8 +63,9 @@ class TDTestCase(TBase): idxv = arg[4] # use db - rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') - self.checkListString(rlist, "Database changed") + if mode != "-R": + rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') + self.checkListString(rlist, "Database changed") # hori cmd = f'{mode} -s "select * from {db}.{stb} limit 10' @@ -86,7 +87,7 @@ class TDTestCase(TBase): self.taos(f'{mode} -B -s "select * from {db}.{stb} where ts < 1"') # get empty result - rlist = self.taos(f'{mode} -s "select * from {db}.{stb} where ts < 1"') + rlist = self.taos(f'{mode} -r -s "select * from {db}.{stb} where ts < 1"') self.checkListString(rlist, "Query OK, 0 row(s) in set") def checkBasic(self): @@ -99,7 +100,7 @@ class TDTestCase(TBase): # native restful websock test args = [ ["", 18, 346, -2, 310], - ["-R -r", 22, 350, -3, 313], + ["-R", 22, 350, -3, 313], ["-T 40 -E http://localhost:6041", 21, 349, -3, 312] ] for arg in args: From e87541efaadc655ae55be51ad8f1bcecff39a85d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Feb 2025 16:18:27 +0800 Subject: [PATCH 47/71] feat:[TD-33798]modify metadata logic for tmq --- source/dnode/vnode/src/tq/tqUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 7dc9a88164..bc8b8504c4 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -389,7 +389,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, code = buildBatchMeta(&btMetaRsp, pHead->msgType, pHead->bodyLen, pHead->body); fetchVer++; if (code != 0){ - continue; + goto END; } totalMetaRows++; if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > pRequest->timeout)) { @@ -423,7 +423,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, taosxRsp.createTableReq = NULL; fetchVer++; if (code != 0){ - continue; + goto END; } totalMetaRows++; if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || From 47e23e3d99ac10d8b376e1f8fcc2965f65be01a1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Feb 2025 17:37:01 +0800 Subject: [PATCH 48/71] feat:[TS-5613]support bool in castfsd --- source/libs/parser/inc/sql.y | 4 ++-- tests/system-test/2-query/test_ts4467.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 42d2e95d24..ca0a806037 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1285,9 +1285,9 @@ function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= cols_func(B) NK_LP cols_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= - CAST(B) NK_LP expr_or_subquery(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } + CAST(B) NK_LP common_expression(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } function_expression(A) ::= - CAST(B) NK_LP expr_or_subquery(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } + CAST(B) NK_LP common_expression(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } function_expression(A) ::= POSITION(B) NK_LP expr_or_subquery(C) IN expr_or_subquery(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createPositionFunctionNode(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); } function_expression(A) ::= diff --git a/tests/system-test/2-query/test_ts4467.py b/tests/system-test/2-query/test_ts4467.py index 4e09a9fbbe..b596c117fd 100644 --- a/tests/system-test/2-query/test_ts4467.py +++ b/tests/system-test/2-query/test_ts4467.py @@ -55,6 +55,11 @@ class TDTestCase: tdLog.debug("res2: %s" % str(res2)) assert(len(res1) == len(res2) and res1[0][0] == res2[4][0]) + # test ts-5613 + sql = "select cast(2<>3 as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From ee0f7a4d38c20f27323904013781405ac9f8ea0b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Feb 2025 17:45:07 +0800 Subject: [PATCH 49/71] feat:[TS-5613]support bool in cast --- source/libs/parser/inc/sql.y | 4 ++-- tests/system-test/2-query/test_ts4467.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index ca0a806037..818ef6cf2c 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1454,13 +1454,13 @@ boolean_value_expression(A) ::= NOT(C) boolean_primary(B). A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); } boolean_value_expression(A) ::= - boolean_value_expression(B) OR boolean_value_expression(C). { + expr_or_subquery(B) OR expr_or_subquery(C). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, C); A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); } boolean_value_expression(A) ::= - boolean_value_expression(B) AND boolean_value_expression(C). { + expr_or_subquery(B) AND expr_or_subquery(C). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, C); A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); diff --git a/tests/system-test/2-query/test_ts4467.py b/tests/system-test/2-query/test_ts4467.py index b596c117fd..02ee2f0bbc 100644 --- a/tests/system-test/2-query/test_ts4467.py +++ b/tests/system-test/2-query/test_ts4467.py @@ -60,6 +60,18 @@ class TDTestCase: tdSql.query(sql) tdSql.checkData(0,0,1) + sql = "select cast(2 not in(3) as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,1) + + sql = "select cast(2 is NULL as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,0) + + sql = "select cast(2 and 1 as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From c017afb8325ffd1ad199d126e97e1ec6ae8d6dd4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Feb 2025 18:14:12 +0800 Subject: [PATCH 50/71] docs: minor changes --- docs/en/14-reference/09-error-code.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 5ef84118ca..2d75dc953c 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -460,7 +460,7 @@ This document details the server error codes that may be encountered when using | 0x80002667 | Invalid usage of expr: %s | Illegal expression | Check and correct the SQL statement | | 0x80002687 | True_for duration cannot be negative | Use negative value as true_for duration | Check and correct the SQL statement | | 0x80002688 | Cannot use 'year' or 'month' as true_for duration | Use year or month as true_for_duration | Check and correct the SQL statement | -| 0x80002680 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement | +| 0x80002689 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement | | 0x8000268A | Cols function's first param must be a select function that output a single row | The first parameter of the cols function should be a selection function | Check and correct the SQL statement | | 0x8000268B | Invalid using cols function with multiple output columns | Illegal using the cols function for multiple column output | Check and correct the SQL statement | | 0x8000268C | Invalid using alias for cols function | Illegal cols function alias | Check and correct the SQL statement | From 18593a06efca5459167a75d76c3906706d9c77ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Feb 2025 18:21:20 +0800 Subject: [PATCH 51/71] fix: ci errors --- tests/pytest/util/tserror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/tserror.py b/tests/pytest/util/tserror.py index 6dca0012ef..ba9b78e8eb 100644 --- a/tests/pytest/util/tserror.py +++ b/tests/pytest/util/tserror.py @@ -16,4 +16,4 @@ TSDB_CODE_TSC_INTERNAL_ERROR = (TAOS_DEF_ERROR_CODE | 0x02FF) TSDB_CODE_PAR_SYNTAX_ERROR = (TAOS_DEF_ERROR_CODE | 0x2600) -TSDB_CODE_PAR_INVALID_COLS_FUNCTION = (TAOS_DEF_ERROR_CODE | 0x2687) +TSDB_CODE_PAR_INVALID_COLS_FUNCTION = (TAOS_DEF_ERROR_CODE | 0x2689) From 1c771c579eff618bd20130345dd6b390b725bf1c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 09:19:20 +0800 Subject: [PATCH 52/71] fix: add source.sql --- tests/army/cmdline/data/source.sql | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/army/cmdline/data/source.sql diff --git a/tests/army/cmdline/data/source.sql b/tests/army/cmdline/data/source.sql new file mode 100644 index 0000000000..e546d17ace --- /dev/null +++ b/tests/army/cmdline/data/source.sql @@ -0,0 +1,37 @@ +drop database if exists db; +create database db ; +create table db.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24)); +create table db.d0 using db.meters tags(1, 'china'); +insert into db.d0 values(1700000000001,1,1,1); +insert into db.d0 values(1700000000002,2,2,2); +insert into db.d0 values(1700000000003,3,3,4); +insert into db.d0 values(1700000000004,4,4,4); +insert into db.d0 values(1700000000005,5,5,5); +show db.tables; +set max_binary_display_width 100; +explain select * from db.meters; +show dnodes; +desc db.d0; +select * from db.meters limit 2; +drop database if exists dbus; +create database dbus precision 'us'; +create table dbus.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24)); +create table dbus.d0 using dbus.meters tags(1, 'china'); +insert into dbus.d0 values(1700000000000001,1,1,1); +insert into dbus.d0 values(1700000000000002,2,2,2); +insert into dbus.d0 values(1700000000000003,3,3,3); +insert into dbus.d0 values(1700000000000004,4,4,4); +insert into dbus.d0 values(1700000000000005,5,5,5); +select * from dbus.meters; +select * from dbus.meters\G; +drop database if exists duns; +create database duns precision 'ns'; +create table duns.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24)); +create table duns.d0 using duns.meters tags(1, 'china'); +insert into duns.d0 values(1700000000000000001,1,1,1); +insert into duns.d0 values(1700000000000000002,2,2,2); +insert into duns.d0 values(1700000000000000003,3,3,4); +insert into duns.d0 values(1700000000000000004,4,4,4); +insert into duns.d0 values(1700000000000000005,5,5,5); +select * from duns.meters; +select * from duns.meters\G; \ No newline at end of file From 50e88765b5bb5d05f9abffa87ba75171076e643d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 09:26:36 +0800 Subject: [PATCH 53/71] fix: export csv no need semi-colon --- tests/army/cmdline/taosCli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 8e2c05c166..ba52ba97be 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -110,7 +110,7 @@ class TDTestCase(TBase): def checkDumpInOutMode(self, source, arg, db, insertRows): mode = arg[0] self.taos(f'{mode} -s "source {source}" ') - self.taos(f'{mode} -s "select * from {db}.d0; >>d0.csv" ') + self.taos(f'{mode} -s "select * from {db}.d0>>d0.csv" ') # use db rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') From 9b6a094cac90768eac9b42d1bd2a3ac446f1fbf0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 25 Feb 2025 09:32:17 +0800 Subject: [PATCH 54/71] feat:[TS-5613] modify sql.y to move bool_expression to common expression --- source/libs/parser/inc/sql.y | 8 ++++++-- tests/system-test/2-query/test_ts4467.py | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 818ef6cf2c..e7c0617d88 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -50,10 +50,14 @@ %left AND. %left UNION ALL MINUS EXCEPT INTERSECT. %left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. +%left NK_LT NK_GT NK_LE NK_GE NK_EQ NK_NE LIKE MATCH NMATCH REGEXP CONTAINS BETWEEN IS IN. %left NK_PLUS NK_MINUS. %left NK_STAR NK_SLASH NK_REM. %left NK_CONCAT. +%right NOT. +%left NK_ARROW. + %right INNER LEFT RIGHT FULL OUTER SEMI ANTI ASOF WINDOW JOIN ON WINDOW_OFFSET JLIMIT. /************************************************ create/alter account *****************************************/ @@ -1454,13 +1458,13 @@ boolean_value_expression(A) ::= NOT(C) boolean_primary(B). A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); } boolean_value_expression(A) ::= - expr_or_subquery(B) OR expr_or_subquery(C). { + boolean_value_expression(B) OR boolean_value_expression(C). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, C); A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); } boolean_value_expression(A) ::= - expr_or_subquery(B) AND expr_or_subquery(C). { + boolean_value_expression(B) AND boolean_value_expression(C). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, C); A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); diff --git a/tests/system-test/2-query/test_ts4467.py b/tests/system-test/2-query/test_ts4467.py index 02ee2f0bbc..52ea3aec70 100644 --- a/tests/system-test/2-query/test_ts4467.py +++ b/tests/system-test/2-query/test_ts4467.py @@ -68,10 +68,6 @@ class TDTestCase: tdSql.query(sql) tdSql.checkData(0,0,0) - sql = "select cast(2 and 1 as int) from t" - tdSql.query(sql) - tdSql.checkData(0,0,1) - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 100aa30eab37244f7c9f2a26fb1016b33186601f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 13:06:05 +0800 Subject: [PATCH 55/71] fix: commandline.py for taosdump --- tests/army/frame/caseBase.py | 11 ++- .../taosdump/native/json/insertFullType.json | 70 +++++++++++++++++++ .../taosdump/native/taosdumpCommandline.py | 54 ++++++++++---- 3 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 tests/army/tools/taosdump/native/json/insertFullType.json diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 649bc231b0..cdc0025470 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -282,8 +282,13 @@ class TBase: # # run bin file # - def taos(self, command, show = True): - return frame.etool.runBinFile("taos", command, show) + # taos + def taos(self, command, show = True, checkRun = False): + return frame.etool.runBinFile("taos", command, show, checkRun) + + def taosdump(self, command, show = True, checkRun = True): + return frame.etool.runBinFile("taosdump", command, show, checkRun) + # # util @@ -420,7 +425,7 @@ class TBase: if vgroups != None: tdSql.checkData(0, 0, vgroups) - return db, stb,child_count, insert_rows + return db, stb, child_count, insert_rows # tmq diff --git a/tests/army/tools/taosdump/native/json/insertFullType.json b/tests/army/tools/taosdump/native/json/insertFullType.json new file mode 100644 index 0000000000..5bdecef72b --- /dev/null +++ b/tests/army/tools/taosdump/native/json/insertFullType.json @@ -0,0 +1,70 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 2, + "insert_rows":100, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} + ], + "tags":[ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} + ] + } + ] + } + ] +} diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index 2779ad57d2..837b48ff18 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -97,9 +97,9 @@ class TDTestCase(TBase): self.exec(cmd) self.checkCorrectWithJson(jsonFile) - def insertData(self, benchmark, json, db): + def insertData(self, json): # insert super table - self.testBenchmarkJson(benchmark, json) + db, stb, child_count, insert_rows = self.insertBenchJson(json) # normal table sqls = [ @@ -112,6 +112,8 @@ class TDTestCase(TBase): ] for sql in sqls: tdSql.execute(sql) + + return db, stb, child_count, insert_rows def dumpOut(self, taosdump, db , outdir): # dump out @@ -152,10 +154,35 @@ class TDTestCase(TBase): # basic commandline - def basicCommandLine(self, taosdump, tmpdir): - # -h -P -u -p -o - self.exec(taosdump + f" -h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}") - self.clearPath(tmpdir) + def basicCommandLine(self, tmpdir): + #command and check result + checkItems = [ + f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"], + f"-r result -gg -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"], + f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"], + f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"], + f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"], + f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], + f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], + f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], + f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], + f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"], + f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"], + f"--help", ["Report bugs to"], + f"-?", ["Report bugs to"], + f"-V", ["version:"], + f"--usage", ["taosdump [OPTION...] -o outpath"] + ] + + # executes + for item in checkItems: + command = item[0] + results = item[1] + rlist = self.taosdump(command) + for result in results: + self.checkListString(result) + # clear tmp + self.clearPath(tmpdir) # check except def checkExcept(self, command): @@ -178,28 +205,29 @@ class TDTestCase(TBase): self.checkExcept(taosdump + " -A -o ") self.checkExcept(taosdump + " -A -o ./noexistpath/") self.checkExcept(taosdump + f" -d invalidAVRO -o {tmpdir}") + self.checkExcept(taosdump + f" -d unknown -o {tmpdir}") self.checkExcept(taosdump + f" -P invalidport") self.checkExcept(taosdump + f" -D") self.checkExcept(taosdump + f" -P 65536") + self.checkExcept(taosdump + f" -t 2 -k 2 -z 1 -C https://not-exist.com:80/cloud -D test -o {tmpdir}") + self.checkExcept(taosdump + f" -P 65536") # run def run(self): - # database - db = "pridb" - newdb = "npridb" # find taosdump, benchmark, tmpdir = self.findPrograme() - json = "./tools/taosdump/ws/json/primaryKey.json" + json = "./tools/taosdump/native/json/insertFullType.json" # insert data with taosBenchmark - self.insertData(benchmark, json, db) + db, stb, child_count, insert_rows = self.insertData(benchmark, json) + newdb = "new" + db # basic commandline - self.basicCommandLine(taosdump, tmpdir) + self.basicCommandLine(db, stb, tmpdir) # except commandline - self.exceptCommandLine(taosdump, tmpdir) + self.exceptCommandLine(db, stb, tmpdir) # dump out #self.dumpOut(taosdump, db, tmpdir) From aff92b763c4a8b5b003af9e051b5fd00be6f7b89 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 14:02:38 +0800 Subject: [PATCH 56/71] fix: support error return to vlist --- tests/army/frame/eos.py | 10 +++-- tests/army/frame/etool.py | 4 +- .../taosdump/native/taosdumpCommandline.py | 40 +++++++++---------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 9e502c3e59..2d07fc5ffe 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -89,12 +89,16 @@ def run(command, show = True): # return list after run -def runRetList(command, show = True, checkRun = False): +def runRetList(command, show = True, checkRun = False, retFail = False): output, error, code = run(command, show) if checkRun and code != 0: print(f"eos.runRetList checkRun return code failed. code={code} error={error}") - assert code == 0 - return output.splitlines() + + assert code == 0 + rList = output.splitlines() + if retFail and error != "": + rList += error.splitlines() + return rList # # file diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index 164e5e127d..4ad3efb036 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -66,7 +66,7 @@ def curFile(fullPath, filename): # run build/bin file -def runBinFile(fname, command, show = True, checkRun = False): +def runBinFile(fname, command, show = True, checkRun = False, retFail = False ): binFile = frame.epath.binFile(fname) if frame.eos.isWin(): binFile += ".exe" @@ -74,7 +74,7 @@ def runBinFile(fname, command, show = True, checkRun = False): cmd = f"{binFile} {command}" if show: tdLog.info(cmd) - return frame.eos.runRetList(cmd, show, checkRun) + return frame.eos.runRetList(cmd, show, checkRun, retFail) # exe build/bin file def exeBinFile(fname, command, wait=True, show=True): diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index 837b48ff18..21562c15e0 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -157,21 +157,21 @@ class TDTestCase(TBase): def basicCommandLine(self, tmpdir): #command and check result checkItems = [ - f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"], - f"-r result -gg -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"], - f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"], - f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"], - f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"], - f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], - f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], - f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], - f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"], - f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"], - f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"], - f"--help", ["Report bugs to"], - f"-?", ["Report bugs to"], - f"-V", ["version:"], - f"--usage", ["taosdump [OPTION...] -o outpath"] + [f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"]], + [f"-r result -gg -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]], + [f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"]], + [f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"]], + [f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"]], + [f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]], + [f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"]], + [f"--help", ["Report bugs to"]], + [f"-?", ["Report bugs to"]], + [f"-V", ["version:"]], + [f"--usage", ["taosdump [OPTION...] -o outpath"]] ] # executes @@ -180,7 +180,7 @@ class TDTestCase(TBase): results = item[1] rlist = self.taosdump(command) for result in results: - self.checkListString(result) + self.checkListString(rlist, result) # clear tmp self.clearPath(tmpdir) @@ -197,7 +197,7 @@ class TDTestCase(TBase): # except commandline - def exceptCommandLine(self, taosdump, tmpdir): + def exceptCommandLine(self, taosdump, db, stb, tmpdir): # -o self.checkExcept(taosdump + " -o= ") self.checkExcept(taosdump + " -o") @@ -220,14 +220,14 @@ class TDTestCase(TBase): json = "./tools/taosdump/native/json/insertFullType.json" # insert data with taosBenchmark - db, stb, child_count, insert_rows = self.insertData(benchmark, json) + db, stb, child_count, insert_rows = self.insertData(json) newdb = "new" + db # basic commandline - self.basicCommandLine(db, stb, tmpdir) + self.basicCommandLine(tmpdir) # except commandline - self.exceptCommandLine(db, stb, tmpdir) + self.exceptCommandLine(taosdump, db, stb, tmpdir) # dump out #self.dumpOut(taosdump, db, tmpdir) From f30eac127281e77c751e9e140b3259526f1fc349 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 14:06:19 +0800 Subject: [PATCH 57/71] fix: caseBase.taosdump retFail default is TRue --- tests/army/frame/caseBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index cdc0025470..4427ddcea5 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -286,8 +286,8 @@ class TBase: def taos(self, command, show = True, checkRun = False): return frame.etool.runBinFile("taos", command, show, checkRun) - def taosdump(self, command, show = True, checkRun = True): - return frame.etool.runBinFile("taosdump", command, show, checkRun) + def taosdump(self, command, show = True, checkRun = True, retFail = True): + return frame.etool.runBinFile("taosdump", command, show, checkRun, retFail) # From e441bd3f625b723a88340eb234295fb96e710564 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 14:34:57 +0800 Subject: [PATCH 58/71] fix: taosdumpCommandline.py passed --- tests/army/frame/eos.py | 2 +- .../army/tools/taosdump/native/taosdumpCommandline.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 2d07fc5ffe..9994fa7497 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -93,8 +93,8 @@ def runRetList(command, show = True, checkRun = False, retFail = False): output, error, code = run(command, show) if checkRun and code != 0: print(f"eos.runRetList checkRun return code failed. code={code} error={error}") + assert code == 0 - assert code == 0 rList = output.splitlines() if retFail and error != "": rList += error.splitlines() diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index 21562c15e0..e93d938dac 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -157,15 +157,15 @@ class TDTestCase(TBase): def basicCommandLine(self, tmpdir): #command and check result checkItems = [ - [f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"]], - [f"-r result -gg -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]], - [f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"]], - [f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 200 row(s) dumped out!"]], + [f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], + [f"-r result -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]], + [f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], + [f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], [f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"]], [f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], [f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], [f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], - [f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]], [f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]], [f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"]], [f"--help", ["Report bugs to"]], From 33f4a4926daebb391d7cfa0ee9bd365edfba0ec8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 15:20:41 +0800 Subject: [PATCH 59/71] modify taos CopyRight from 2023 to 2025 --- tools/shell/src/shellArguments.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index ba210850eb..93381b4f51 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -443,7 +443,7 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); shell.info.clientVersion = "Welcome to the %s Command Line Interface, Client Version:%s\r\n" - "Copyright (c) 2023 by %s, all rights reserved.\r\n\r\n"; + "Copyright (c) 2025 by %s, all rights reserved.\r\n\r\n"; #ifdef CUS_NAME strcpy(shell.info.cusName, CUS_NAME); #else From 1ed9efee23b282e71fb669a2151909863bcc4bd5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Feb 2025 10:59:49 +0800 Subject: [PATCH 60/71] fix: add some ut --- include/os/osSysinfo.h | 2 + source/os/src/osRand.c | 2 +- source/os/src/osSignal.c | 2 + source/os/src/osString.c | 4 +- source/os/src/osSysinfo.c | 5 +- source/os/test/CMakeLists.txt | 7 - source/os/test/osSignalTests.cpp | 40 -- source/os/test/osStringTests.cpp | 943 ++++++++++++++++++------------- source/os/test/osTests.cpp | 369 +++++++++++- source/os/test/osThreadTests.cpp | 40 +- 10 files changed, 938 insertions(+), 476 deletions(-) delete mode 100644 source/os/test/osSignalTests.cpp diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 1f4dfe3809..5e1b889580 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -47,8 +47,10 @@ int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetSysAvailMemory(int64_t *availSize); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); +int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); void taosSetDefaultProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); +int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes); int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 0bf67f96a0..d13342ca28 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -87,7 +87,7 @@ void taosRandStr(char* str, int32_t size) { void taosRandStr2(char* str, int32_t size) { const char* set = "abcdefghijklmnopqrstuvwxyz0123456789@"; - int32_t len = strlen(set); + int32_t len = 37; for (int32_t i = 0; i < size; ++i) { str[i] = set[taosRand() % len]; diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index f341f5422d..238093e418 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -114,6 +114,7 @@ int32_t taosDflSignal(int32_t signum) { return 0; } +#if 0 int32_t taosKillChildOnParentStopped() { #ifndef _TD_DARWIN_64 int32_t code = prctl(PR_SET_PDEATHSIG, SIGKILL); @@ -126,5 +127,6 @@ int32_t taosKillChildOnParentStopped() { #endif return 0; } +#endif #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 1d07b64c70..24fd8ca034 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -294,6 +294,7 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { //#endif } +#if 0 int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { if (target_ucs4 == NULL || source_ucs4 == NULL || len_ucs4 <= 0) { return TSDB_CODE_INVALID_PARA; @@ -307,11 +308,12 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) return TSDB_CODE_SUCCESS; } +#endif iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) { if(idx == NULL) { terrno = TSDB_CODE_INVALID_PARA; - return (iconv_t)-1; + return (iconv_t)NULL; } if (charsetCxt == NULL){ charsetCxt = tsCharsetCxt; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 6cb0e00b82..dccce2d987 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -147,7 +147,6 @@ static void taosGetProcIOnfos() { #endif static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - OS_PARAM_CHECK(cpuInfo); int32_t code = 0; #ifdef WINDOWS FILETIME pre_idleTime = {0}; @@ -200,7 +199,6 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - OS_PARAM_CHECK(cpuInfo); int32_t code = 0; #ifdef WINDOWS @@ -541,7 +539,6 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { // Returns the container's CPU quota if successful, otherwise returns the physical CPU cores static int32_t taosCntrGetCpuCores(float *numOfCores) { - OS_PARAM_CHECK(numOfCores); #ifdef WINDOWS return TSDB_CODE_UNSUPPORT_OS; #elif defined(_TD_DARWIN_64) @@ -1092,6 +1089,7 @@ void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes if (transmit_bytes) *transmit_bytes = 0; } +#if 0 void taosKillSystem() { #ifdef WINDOWS printf("function taosKillSystem, exit!"); @@ -1105,6 +1103,7 @@ void taosKillSystem() { (void)kill(tsProcId, 2); #endif } +#endif #define UUIDLEN (36) int32_t taosGetSystemUUIDLimit36(char *uid, int32_t uidlen) { diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index 2ba6b73e29..a26fc88916 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -61,13 +61,6 @@ add_test( COMMAND osSemaphoreTests ) -add_executable(osSignalTests "osSignalTests.cpp") -target_link_libraries(osSignalTests os util gtest_main) -add_test( - NAME osSignalTests - COMMAND osSignalTests -) - add_executable(osSleepTests "osSleepTests.cpp") target_link_libraries(osSleepTests os util gtest_main) add_test( diff --git a/source/os/test/osSignalTests.cpp b/source/os/test/osSignalTests.cpp deleted file mode 100644 index f4cc6e9e58..0000000000 --- a/source/os/test/osSignalTests.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" -#pragma GCC diagnostic ignored "-Wsign-compare" -#pragma GCC diagnostic ignored "-Wformat" -#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" -#pragma GCC diagnostic ignored "-Wpointer-arith" - -#include "os.h" -#include "tlog.h" - -TEST(osSignalTests, taosSetSignal) { - // Set up SIGINT handler using taosSetSignal - //taosSetSignal(SIGINT, sigint_handler); - - // Print PID for testing purposes - // printf("PID: %d\n", getpid()); - - // Wait for signal to be received -} diff --git a/source/os/test/osStringTests.cpp b/source/os/test/osStringTests.cpp index 36f5207346..4e71b4b884 100644 --- a/source/os/test/osStringTests.cpp +++ b/source/os/test/osStringTests.cpp @@ -33,8 +33,8 @@ #ifdef WINDOWS TEST(osStringTests, strsepNormalInput) { char str[] = "This is a test string."; - char * ptr = str; - char * tok = NULL; + char* ptr = str; + char* tok = NULL; const char delim[] = " "; while ((tok = strsep(&ptr, delim)) != NULL) { @@ -59,9 +59,9 @@ TEST(osStringTests, strsepEmptyInput) { } TEST(osStringTests, strsepNullInput) { - char * str = NULL; - char * ptr = str; - char * tok = NULL; + char* str = NULL; + char* ptr = str; + char* tok = NULL; const char delim[] = " "; while ((tok = strsep(&ptr, delim)) != NULL) { @@ -75,7 +75,7 @@ TEST(osStringTests, strsepNullInput) { TEST(osStringTests, strndupNormalInput) { const char s[] = "This is a test string."; int size = strlen(s) + 1; - char * s2 = taosStrndup(s, size); + char* s2 = taosStrndup(s, size); EXPECT_STREQ(s, s2); @@ -114,523 +114,674 @@ TEST(osStringTests, osUcs4lenTests2) { } TEST(osStringTests, ostsnprintfTests) { - char buffer[50] = {0}; - int64_t ret; + char buffer[50] = {0}; + int64_t ret; - ret = tsnprintf(buffer, sizeof(buffer), "Hello, %s!", "World"); - EXPECT_EQ(ret, 13); - EXPECT_STREQ(buffer, "Hello, World!"); + ret = tsnprintf(buffer, sizeof(buffer), "Hello, %s!", "World"); + EXPECT_EQ(ret, 13); + EXPECT_STREQ(buffer, "Hello, World!"); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, 10, "Hello, %s!", "World"); - EXPECT_EQ(ret, 9); - EXPECT_EQ(strncmp(buffer, "Hello, Wo", 9), 0); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, 10, "Hello, %s!", "World"); + EXPECT_EQ(ret, 9); + EXPECT_EQ(strncmp(buffer, "Hello, Wo", 9), 0); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, 10, "Hello%s", "World"); - EXPECT_EQ(ret, 9); - EXPECT_EQ(strncmp(buffer, "HelloWorl", 9), 0); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, 10, "Hello%s", "World"); + EXPECT_EQ(ret, 9); + EXPECT_EQ(strncmp(buffer, "HelloWorl", 9), 0); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, 0, "Hello, %s!", "World"); - EXPECT_EQ(ret, 0); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, 0, "Hello, %s!", "World"); + EXPECT_EQ(ret, 0); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, SIZE_MAX + 1, "Hello, %s!", "World"); - EXPECT_EQ(ret, 0); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, SIZE_MAX + 1, "Hello, %s!", "World"); + EXPECT_EQ(ret, 0); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, sizeof(buffer), ""); - EXPECT_EQ(ret, 0); - EXPECT_STREQ(buffer, ""); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, sizeof(buffer), ""); + EXPECT_EQ(ret, 0); + EXPECT_STREQ(buffer, ""); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, sizeof(buffer), "Number: %d", 42); - EXPECT_EQ(ret, 10); - EXPECT_STREQ(buffer, "Number: 42"); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, sizeof(buffer), "Number: %d", 42); + EXPECT_EQ(ret, 10); + EXPECT_STREQ(buffer, "Number: 42"); - memset(buffer, 0, sizeof(buffer)); - ret = tsnprintf(buffer, sizeof(buffer), "Float: %.2f", 3.14); - EXPECT_EQ(ret, 11); - EXPECT_STREQ(buffer, "Float: 3.14"); + memset(buffer, 0, sizeof(buffer)); + ret = tsnprintf(buffer, sizeof(buffer), "Float: %.2f", 3.14); + EXPECT_EQ(ret, 11); + EXPECT_STREQ(buffer, "Float: 3.14"); } TEST(osStringTests, osStr2Int64) { - int64_t val; - int32_t result; + int64_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2int64(NULL, &val); - assert(result == TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2int64(NULL, &val); + assert(result == TSDB_CODE_INVALID_PARA); - result = taosStr2int64("123", NULL); - ASSERT_NE(result, 0); + result = taosStr2int64("123", NULL); + ASSERT_NE(result, 0); - // 测试无效输入 - result = taosStr2int64("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2int64("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2int64("", &val); - ASSERT_NE(result, 0); + result = taosStr2int64("", &val); + ASSERT_NE(result, 0); - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%lld", LLONG_MAX); - result = taosStr2int64(large_num, &val); - assert(result == 0); - assert(val == LLONG_MAX); + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%lld", LLONG_MAX); + result = taosStr2int64(large_num, &val); + assert(result == 0); + assert(val == LLONG_MAX); - snprintf(large_num, sizeof(large_num), "%lld", LLONG_MIN); - result = taosStr2int64(large_num, &val); - assert(result == 0); - assert(val == LLONG_MIN); + snprintf(large_num, sizeof(large_num), "%lld", LLONG_MIN); + result = taosStr2int64(large_num, &val); + assert(result == 0); + assert(val == LLONG_MIN); - result = taosStr2int64("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2int64("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2int64("abc123", &val); - ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2int64("12345", &val); - assert(result == 0); - assert(val == 12345); + result = taosStr2int64("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2int64("12345", &val); + assert(result == 0); + assert(val == 12345); - result = taosStr2int64("-12345", &val); - assert(result == 0); - assert(val == -12345); + result = taosStr2int64("-12345", &val); + assert(result == 0); + assert(val == -12345); - result = taosStr2int64("0", &val); - assert(result == 0); - assert(val == 0); + result = taosStr2int64("0", &val); + assert(result == 0); + assert(val == 0); - // 测试带空格的字符串 - result = taosStr2int64(" 12345", &val); - assert(result == 0); - assert(val == 12345); + // 测试带空格的字符串 + result = taosStr2int64(" 12345", &val); + assert(result == 0); + assert(val == 12345); - result = taosStr2int64("12345 ", &val); - assert(result == 0); - assert(val == 12345); + result = taosStr2int64("12345 ", &val); + assert(result == 0); + assert(val == 12345); } TEST(osStringTests, osStr2int32) { - int32_t val; - int32_t result; + int32_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2int32(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2int32(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2int32("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2int32("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2int32("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2int32("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2int32("", &val); - ASSERT_NE(result, 0); + result = taosStr2int32("", &val); + ASSERT_NE(result, 0); - // 测试超出范围的值 - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%d", INT_MAX); - result = taosStr2int32(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, INT_MAX); + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT_MAX); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT_MAX); - snprintf(large_num, sizeof(large_num), "%d", INT_MIN); - result = taosStr2int32(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, INT_MIN); + snprintf(large_num, sizeof(large_num), "%d", INT_MIN); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT_MIN); - // 测试大于 INT32 范围的值 - snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MAX + 1); - result = taosStr2int32(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + // 测试大于 INT32 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MAX + 1); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MIN - 1); - result = taosStr2int32(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MIN - 1); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - - result = taosStr2int32("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2int32("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2int32("abc123", &val); - ASSERT_NE(result, 0); + result = taosStr2int32("abc123", &val); + ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2int32("12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + // 测试有效的整数字符串 + result = taosStr2int32("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2int32("-12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, -12345); + result = taosStr2int32("-12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -12345); - result = taosStr2int32("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2int32("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - // 测试带空格的字符串 - result = taosStr2int32(" 12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + // 测试带空格的字符串 + result = taosStr2int32(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2int32("12345 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2int32("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); } TEST(osStringTests, taosStr2int16) { - int16_t val; - int32_t result; + int16_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2int16(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2int16(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2int16("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2int16("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2int16("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2int16("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2int16("", &val); - ASSERT_NE(result, 0); + result = taosStr2int16("", &val); + ASSERT_NE(result, 0); - // 测试超出范围的值 - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%d", INT16_MAX); - result = taosStr2int16(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, INT16_MAX); + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT16_MAX); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT16_MAX); - snprintf(large_num, sizeof(large_num), "%d", INT16_MIN); - result = taosStr2int16(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, INT16_MIN); + snprintf(large_num, sizeof(large_num), "%d", INT16_MIN); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT16_MIN); - // 测试大于 INT16 范围的值 - snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MAX + 1); - result = taosStr2int16(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + // 测试大于 INT16 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MAX + 1); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MIN - 1); - result = taosStr2int16(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MIN - 1); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - result = taosStr2int16("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2int16("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2int16("abc123", &val); - ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2int16("12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2int16("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2int16("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2int16("-12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, -12345); + result = taosStr2int16("-12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -12345); - result = taosStr2int16("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2int16("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - // 测试带空格的字符串 - result = taosStr2int16(" 12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + // 测试带空格的字符串 + result = taosStr2int16(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2int16("12345 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2int16("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); } - TEST(osStringTests, taosStr2int8) { - int8_t val; - int32_t result; + int8_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2int8(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2int8(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2int8("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2int8("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2int8("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2int8("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2int8("", &val); - ASSERT_NE(result, 0); + result = taosStr2int8("", &val); + ASSERT_NE(result, 0); - // 测试超出范围的值 - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%d", INT8_MAX); - result = taosStr2int8(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, INT8_MAX); + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT8_MAX); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT8_MAX); - snprintf(large_num, sizeof(large_num), "%d", INT8_MIN); - result = taosStr2int8(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, INT8_MIN); + snprintf(large_num, sizeof(large_num), "%d", INT8_MIN); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT8_MIN); - // 测试大于 INT8 范围的值 - snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MAX + 1); - result = taosStr2int8(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + // 测试大于 INT8 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MAX + 1); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MIN - 1); - result = taosStr2int8(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MIN - 1); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - result = taosStr2int8("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2int8("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2int8("abc123", &val); - ASSERT_NE(result, 0); - - // 测试有效的整数字符串 - result = taosStr2int8("123", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2int8("abc123", &val); + ASSERT_NE(result, 0); - result = taosStr2int8("-123", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, -123); + // 测试有效的整数字符串 + result = taosStr2int8("123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2int8("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2int8("-123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -123); - // 测试带空格的字符串 - result = taosStr2int8(" 123", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2int8("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - result = taosStr2int8("123 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + // 测试带空格的字符串 + result = taosStr2int8(" 123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("123 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); } TEST(osStringTests, osStr2Uint64) { - uint64_t val; - int32_t result; + uint64_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2Uint64(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2Uint64(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2Uint64("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2Uint64("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2Uint64("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2Uint64("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2Uint64("", &val); - ASSERT_NE(result, 0); + result = taosStr2Uint64("", &val); + ASSERT_NE(result, 0); - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%llu", ULLONG_MAX); - result = taosStr2Uint64(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, ULLONG_MAX); + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%llu", ULLONG_MAX); + result = taosStr2Uint64(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, ULLONG_MAX); - result = taosStr2Uint64("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2Uint64("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2Uint64("abc123", &val); - ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2Uint64("12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2Uint64("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint64("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2Uint64("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2Uint64("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - // 测试带空格的字符串 - result = taosStr2Uint64(" 12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); - - result = taosStr2Uint64("12345 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + // 测试带空格的字符串 + result = taosStr2Uint64(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + result = taosStr2Uint64("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); } TEST(osStringTests, taosStr2Uint32) { - uint32_t val; - int32_t result; + uint32_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2Uint32(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2Uint32(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2Uint32("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2Uint32("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2Uint32("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2Uint32("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2Uint32("", &val); - ASSERT_NE(result, 0); + result = taosStr2Uint32("", &val); + ASSERT_NE(result, 0); - // 测试超出范围的值 - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%u", UINT32_MAX); - result = taosStr2Uint32(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, UINT32_MAX); + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT32_MAX); + result = taosStr2Uint32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT32_MAX); - // 测试大于 UINT32 范围的值 - snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT32_MAX + 1); - result = taosStr2Uint32(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + // 测试大于 UINT32 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT32_MAX + 1); + result = taosStr2Uint32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - result = taosStr2Uint32("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2Uint32("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2Uint32("abc123", &val); - ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2Uint32("12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2Uint32("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint32("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2Uint32("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2Uint32("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - // 测试带空格的字符串 - result = taosStr2Uint32(" 12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + // 测试带空格的字符串 + result = taosStr2Uint32(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2Uint32("12345 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2Uint32("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); } TEST(osStringTests, taosStr2Uint16) { - uint16_t val; - int32_t result; + uint16_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2Uint16(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2Uint16(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2Uint16("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2Uint16("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2Uint16("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2Uint16("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2Uint16("", &val); - ASSERT_NE(result, 0); + result = taosStr2Uint16("", &val); + ASSERT_NE(result, 0); - // 测试超出范围的值 - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%u", UINT16_MAX); - result = taosStr2Uint16(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, UINT16_MAX); + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT16_MAX); + result = taosStr2Uint16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT16_MAX); - // 测试大于 UINT16 范围的值 - snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT16_MAX + 1); - result = taosStr2Uint16(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + // 测试大于 UINT16 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT16_MAX + 1); + result = taosStr2Uint16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - result = taosStr2Uint16("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2Uint16("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2Uint16("abc123", &val); - ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2Uint16("12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2Uint16("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint16("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2Uint16("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2Uint16("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - // 测试带空格的字符串 - result = taosStr2Uint16(" 12345", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + // 测试带空格的字符串 + result = taosStr2Uint16(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); - result = taosStr2Uint16("12345 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 12345); + result = taosStr2Uint16("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); } TEST(osStringTests, taosStr2Uint8) { - uint8_t val; - int32_t result; + uint8_t val; + int32_t result; - // 测试空指针输入 - result = taosStr2Uint8(NULL, &val); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + // 测试空指针输入 + result = taosStr2Uint8(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - result = taosStr2Uint8("123", NULL); - ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + result = taosStr2Uint8("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); - // 测试无效输入 - result = taosStr2Uint8("abc", &val); - ASSERT_NE(result, 0); + // 测试无效输入 + result = taosStr2Uint8("abc", &val); + ASSERT_NE(result, 0); - result = taosStr2Uint8("", &val); - ASSERT_NE(result, 0); + result = taosStr2Uint8("", &val); + ASSERT_NE(result, 0); - // 测试超出范围的值 - char large_num[50]; - snprintf(large_num, sizeof(large_num), "%u", UINT8_MAX); - result = taosStr2Uint8(large_num, &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, UINT8_MAX); + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT8_MAX); + result = taosStr2Uint8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT8_MAX); - // 测试大于 UINT8 范围的值 - snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT8_MAX + 1); - result = taosStr2Uint8(large_num, &val); - ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + // 测试大于 UINT8 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT8_MAX + 1); + result = taosStr2Uint8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); - result = taosStr2Uint8("123abc", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2Uint8("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2Uint8("abc123", &val); - ASSERT_NE(result, 0); - // 测试有效的整数字符串 - result = taosStr2Uint8("123", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2Uint8("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint8("123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2Uint8("0", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 0); + result = taosStr2Uint8("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); - // 测试带空格的字符串 - result = taosStr2Uint8(" 123", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + // 测试带空格的字符串 + result = taosStr2Uint8(" 123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); - result = taosStr2Uint8("123 ", &val); - ASSERT_EQ(result, 0); - ASSERT_EQ(val, 123); + result = taosStr2Uint8("123 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); } +TEST(osStringTests, strint2) { + const char* ret = tstrdup(NULL); + EXPECT_EQ(ret, nullptr); + + ret = taosStrndupi(NULL, 0); + EXPECT_EQ(ret, nullptr); + + char buf[12] = "12345"; + ret = tstrndup(buf, 4); + EXPECT_NE(ret, nullptr); + + int64_t val = 0; + int32_t ret32 = taosStr2int64(NULL, &val); + EXPECT_NE(ret32, 0); + ret32 = taosStr2int64(buf, NULL); + EXPECT_NE(ret32, 0); + + TdUcs4 p1, p2; + int32_t val32 = 0; + ret32 = tasoUcs4Compare(&p1, NULL, val32); + EXPECT_NE(ret32, 0); + ret32 = tasoUcs4Compare(NULL, &p2, val32); + EXPECT_NE(ret32, 0); + + void* retptr = taosAcquireConv(NULL, M2C, NULL); + EXPECT_EQ(retptr, nullptr); + + ret32 = taosUcs4ToMbs(NULL, 0, NULL, NULL); + EXPECT_EQ(ret32, 0); + ret32 = taosUcs4ToMbs(NULL, -1, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosUcs4ToMbs(NULL, 1, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosUcs4ToMbs(&p1, 1, NULL, NULL); + EXPECT_NE(ret32, 0); + + bool retb = taosValidateEncodec(NULL); + EXPECT_FALSE(retb); + + ret32 = taosUcs4len(NULL); + EXPECT_EQ(ret32, 0); + + unsigned char src[24] = "1234"; + char dst[24] = {0}; + int32_t len = 5; + int32_t bufSize = 24; + ret32 = taosHexEncode(NULL, NULL, len, -1); + EXPECT_NE(ret32, 0); + ret32 = taosHexEncode(src, NULL, len, -1); + EXPECT_NE(ret32, 0); + ret32 = taosHexEncode(src, dst, len, -1); + EXPECT_NE(ret32, 0); + ret32 = taosHexEncode(src, dst, len, bufSize); + EXPECT_EQ(ret32, 0); + + char dst2[24] = {0}; + ret32 = taosHexDecode(NULL, NULL, 0); + EXPECT_NE(ret32, 0); + ret32 = taosHexDecode(NULL, dst2, 0); + EXPECT_NE(ret32, 0); + ret32 = taosHexDecode(dst, NULL, 0); + EXPECT_NE(ret32, 0); + ret32 = taosHexDecode(dst, dst2, 24); + EXPECT_EQ(ret32, 0); + EXPECT_STREQ((char*)src, dst2); +} + +TEST(osStringTests, wchartest) { + char src[24] = "1234"; + TdWchar dst[24] = {0}; + + int32_t ret32 = taosWcharsWidth(NULL, 0); + EXPECT_LT(ret32, 0); + ret32 = taosWcharsWidth(dst, 0); + EXPECT_LT(ret32, 0); + + ret32 = taosMbToWchar(NULL, NULL, 0); + EXPECT_LT(ret32, 0); + ret32 = taosMbToWchar(dst, NULL, 0); + EXPECT_LT(ret32, 0); + ret32 = taosMbToWchar(dst, src, 0); + EXPECT_LT(ret32, 0); + ret32 = taosMbToWchar(dst, src, 4); + EXPECT_GT(ret32, 0); + ret32 = taosWcharsWidth(dst, ret32); + EXPECT_GT(ret32, 0); + + ret32 = taosMbsToWchars(NULL, NULL, 0); + EXPECT_LT(ret32, 0); + ret32 = taosMbsToWchars(dst, NULL, 0); + EXPECT_LT(ret32, 0); + ret32 = taosMbsToWchars(dst, src, 0); + EXPECT_LT(ret32, 0); + ret32 = taosMbsToWchars(dst, src, 4); + EXPECT_GT(ret32, 0); + ret32 = taosWcharsWidth(dst, ret32); + EXPECT_GT(ret32, 0); + + ret32 = taosWcharsWidth(NULL, dst[0]); + EXPECT_NE(ret32, 0); + ret32 = taosWcharToMb(src, dst[0]); + EXPECT_NE(ret32, 0); +} + +TEST(osStringTests, strtransform) { + char src[12] = "12"; + + void* retptr = taosStrCaseStr(NULL, NULL); + EXPECT_EQ(retptr, nullptr); + retptr = taosStrCaseStr(src, NULL); + EXPECT_NE(retptr, nullptr); + + int64_t ret64 = taosStr2Int64(NULL, NULL, 0); + EXPECT_EQ(ret64, 0); + uint64_t retu64 = taosStr2UInt64(NULL, NULL, 0); + EXPECT_EQ(retu64, 0); + int32_t ret32 = taosStr2Int32(NULL, NULL, 0); + EXPECT_EQ(ret32, 0); + uint32_t retu32 = taosStr2UInt32(NULL, NULL, 0); + EXPECT_EQ(retu32, 0); + ret32 = taosStr2Int16(NULL, NULL, 0); + EXPECT_EQ(ret32, 0); + ret32 = taosStr2UInt16(NULL, NULL, 0); + EXPECT_EQ(ret32, 0); + ret32 = taosStr2Int8(NULL, NULL, 0); + EXPECT_EQ(ret32, 0); + ret32 = taosStr2UInt8(NULL, NULL, 0); + EXPECT_EQ(ret32, 0); + + double retd = taosStr2Double(NULL, NULL); + EXPECT_EQ((int32_t)retd, 0); + float retf = taosStr2Float(NULL, NULL); + EXPECT_EQ((int32_t)retf, 0); + + bool retb = isValidateHex(NULL, 0); + EXPECT_FALSE(retb); + + char z[12] = {0}; + ret32 = taosHex2Ascii(NULL, 0, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosHex2Ascii(z, 0, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosHex2Ascii(z, 0, (void**)&z, NULL); + EXPECT_NE(ret32, 0); + + ret64 = tsnprintf(NULL, 0, NULL); + EXPECT_EQ(ret64, 0); + ret64 = tsnprintf(z, 4, NULL); + EXPECT_EQ(ret64, 0); + ret64 = tsnprintf(z, 0, "ab"); + EXPECT_EQ(ret64, 0); + ret64 = tsnprintf(z, 1, "ab"); + EXPECT_EQ(ret64, 0); +} diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 2c20348608..f4e7eff323 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -14,8 +14,8 @@ */ #include -#include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -36,6 +36,230 @@ #include +TEST(osTest, locale) { + char *ret = taosCharsetReplace(NULL); + EXPECT_EQ(ret, nullptr); + + ret = taosCharsetReplace("utf8"); + EXPECT_NE(ret, nullptr); + + ret = taosCharsetReplace("utf-8"); + EXPECT_NE(ret, nullptr); + + taosGetSystemLocale(NULL, ""); + taosGetSystemLocale("", NULL); +} + +TEST(osTest, memory) { + int32_t ret = taosMemoryDbgInitRestore(); + EXPECT_EQ(ret, 0); + + int64_t ret64 = taosMemSize(NULL); + EXPECT_EQ(ret64, 0); +} + +TEST(osTest, rand2) { + char str[128] = {0}; + taosRandStr2(str, 100); +} + +TEST(osTest, socket2) { + int32_t ret32 = taosCloseSocket(NULL); + EXPECT_NE(ret32, 0); + + ret32 = taosSetSockOpt(NULL, 0, 0, NULL, 0); + EXPECT_NE(ret32, 0); + +#if defined(LINUX) + struct in_addr ipInt; + ipInt.s_addr = htonl(0x7F000001); + char buf[128] = {0}; + taosInetNtop(ipInt, buf, 32); +#endif + + ret32 = taosGetIpv4FromFqdn("localhost", NULL); + EXPECT_NE(ret32, 0); + uint32_t ip = 0; + ret32 = taosGetIpv4FromFqdn(NULL, &ip); + EXPECT_NE(ret32, 0); + + taosInetNtoa(NULL, ip); + ret32 = taosInetAddr(NULL); + EXPECT_EQ(ret32, 0); + + ret32 = taosWinSocketInit(); + EXPECT_EQ(ret32, 0); +} + +TEST(osTest, time2) { + taosGetLocalTimezoneOffset(); + + char buf[12] = {0}; + char fmt[12] = {0}; + void *retptr = taosStrpTime(NULL, fmt, NULL); + EXPECT_EQ(retptr, nullptr); + retptr = taosStrpTime(buf, NULL, NULL); + EXPECT_EQ(retptr, nullptr); + + size_t ret = taosStrfTime(NULL, 0, fmt, NULL); + EXPECT_EQ(ret, 0); + ret = taosStrfTime(buf, 0, NULL, NULL); + EXPECT_EQ(ret, 0); + + time_t tp = {0}; + struct tm *retptr2 = taosGmTimeR(&tp, NULL); + EXPECT_EQ(retptr2, nullptr); + retptr2 = taosGmTimeR(NULL, NULL); + EXPECT_EQ(retptr2, nullptr); + + time_t rett = taosTimeGm(NULL); + EXPECT_EQ(rett, -1); + + timezone_t tz = {0}; + retptr2 = taosLocalTime(&tp, NULL, NULL, 0, tz); + EXPECT_EQ(retptr2, nullptr); + retptr2 = taosLocalTime(NULL, NULL, NULL, 0, tz); + EXPECT_EQ(retptr2, nullptr); +} + +TEST(osTest, system) { +#if defined(LINUX) + taosSetConsoleEcho(false); + taosSetConsoleEcho(true); + + taosGetOldTerminalMode(); + taosCloseCmd(NULL); + + TdCmdPtr ptr = taosOpenCmd(NULL); + EXPECT_EQ(ptr, nullptr); + taosCloseCmd(&ptr); + + ptr = taosOpenCmd("echo 'hello world'"); + ASSERT_NE(ptr, nullptr); + + char buf[256] = {0}; + int64_t ret64 = taosGetsCmd(NULL, 0, NULL); + EXPECT_LE(ret64, 0); + ret64 = taosGetsCmd(ptr, 0, NULL); + EXPECT_LE(ret64, 0); + ret64 = taosGetsCmd(ptr, 255, buf); + EXPECT_GT(ret64, 0); + taosCloseCmd(&ptr); + + ptr = taosOpenCmd("echoxxx 'hello world'"); + ASSERT_NE(ptr, nullptr); + ret64 = taosGetsCmd(ptr, 255, buf); + EXPECT_LE(ret64, 0); + taosCloseCmd(&ptr); + + ret64 = taosGetLineCmd(NULL, NULL); + EXPECT_LE(ret64, 0); + ret64 = taosGetLineCmd(ptr, NULL); + EXPECT_LE(ret64, 0); + + ptr = taosOpenCmd("echo 'hello world'"); + ASSERT_NE(ptr, nullptr); + char *ptrBuf = NULL; + ret64 = taosGetLineCmd(ptr, &ptrBuf); + EXPECT_GE(ret64, 0); + taosCloseCmd(&ptr); + + ptr = taosOpenCmd("echoxxx 'hello world'"); + ASSERT_NE(ptr, nullptr); + ret64 = taosGetLineCmd(ptr, &ptrBuf); + EXPECT_LE(ret64, 0); + taosCloseCmd(&ptr); + + int32_t ret32 = taosEOFCmd(NULL); + EXPECT_EQ(ret32, 0); + +#endif +} + +TEST(osTest, sysinfo) { +#if defined(LINUX) + + int32_t ret32 = 0; + + ret32 = taosGetEmail(NULL, 0); + EXPECT_NE(ret32, 0); + + ret32 = taosGetOsReleaseName(NULL, NULL, NULL, 0); + EXPECT_NE(ret32, 0); + + char buf[128] = {0}; + float numOfCores = 0; + ret32 = taosGetCpuInfo(buf, 0, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetCpuInfo(NULL, 0, &numOfCores); + EXPECT_NE(ret32, 0); + + + ret32 = taosGetCpuCores(NULL, false); + EXPECT_NE(ret32, 0); + ret32 = taosGetTotalMemory(NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcMemory(NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetSysMemory(NULL); + EXPECT_NE(ret32, 0); + + ret32 = taosGetDiskSize(buf, NULL); + EXPECT_NE(ret32, 0); + SDiskSize disksize = {0}; + ret32 = taosGetDiskSize(NULL, &disksize); + EXPECT_NE(ret32, 0); + + int64_t tmp = 0; + ret32 = taosGetProcIO(NULL, NULL, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcIO(&tmp, NULL, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcIO(&tmp, &tmp, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcIO(&tmp, &tmp, &tmp, NULL); + EXPECT_NE(ret32, 0); + + ret32 = taosGetProcIODelta(NULL, NULL, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcIODelta(&tmp, NULL, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcIODelta(&tmp, &tmp, NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetProcIODelta(&tmp, &tmp, &tmp, NULL); + EXPECT_NE(ret32, 0); + + taosGetProcIODelta(NULL, NULL, NULL, NULL); + taosGetProcIODelta(&tmp, &tmp, &tmp, &tmp); + + ret32 = taosGetCardInfo(NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetCardInfo(&tmp, NULL); + EXPECT_NE(ret32, 0); + + ret32 = taosGetCardInfoDelta(NULL, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetCardInfoDelta(&tmp, NULL); + EXPECT_NE(ret32, 0); + + taosGetCardInfoDelta(NULL, NULL); + taosGetCardInfoDelta(&tmp, &tmp); + + ret32 = taosGetSystemUUIDLimit36(NULL, 0); + EXPECT_NE(ret32, 0); + + ret32 = taosGetSystemUUIDLen(NULL, 0); + EXPECT_NE(ret32, 0); + ret32 = taosGetSystemUUIDLen(buf, -1); + EXPECT_NE(ret32, 0); + + taosSetCoreDump(false); + + ret32 = taosGetlocalhostname(NULL, 0); + EXPECT_NE(ret32, 0); +#endif +} + TEST(osTest, osFQDNSuccess) { char fqdn[TD_FQDN_LEN]; char ipString[INET_ADDRSTRLEN]; @@ -47,8 +271,8 @@ TEST(osTest, osFQDNSuccess) { struct in_addr addr; addr.s_addr = htonl(ipv4); (void)snprintf(ipString, INET_ADDRSTRLEN, "%u.%u.%u.%u", (unsigned int)(addr.s_addr >> 24) & 0xFF, - (unsigned int)(addr.s_addr >> 16) & 0xFF, (unsigned int)(addr.s_addr >> 8) & 0xFF, - (unsigned int)(addr.s_addr) & 0xFF); + (unsigned int)(addr.s_addr >> 16) & 0xFF, (unsigned int)(addr.s_addr >> 8) & 0xFF, + (unsigned int)(addr.s_addr) & 0xFF); (void)printf("fqdn:%s ip:%s\n", fqdn, ipString); } @@ -56,7 +280,7 @@ TEST(osTest, osFQDNFailed) { char fqdn[1024] = "fqdn_test_not_found"; char ipString[24]; uint32_t ipv4 = 0; - int32_t code = taosGetIpv4FromFqdn(fqdn, &ipv4); + int32_t code = taosGetIpv4FromFqdn(fqdn, &ipv4); ASSERT_NE(code, 0); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -79,7 +303,7 @@ TEST(osTest, osSystem) { } void fileOperateOnFree(void *param) { - char * fname = (char *)param; + char *fname = (char *)param; TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); (void)printf("On free thread open file\n"); ASSERT_NE(pFile, nullptr); @@ -101,7 +325,7 @@ void *fileOperateOnFreeThread(void *param) { return NULL; } void fileOperateOnBusy(void *param) { - char * fname = (char *)param; + char *fname = (char *)param; TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); (void)printf("On busy thread open file\n"); if (pFile == NULL) return; @@ -165,9 +389,9 @@ TEST(osTest, osFile) { (void)taosThreadJoin(thread2, NULL); taosThreadClear(&thread2); - //int ret = taosRemoveFile(fname); - //ASSERT_EQ(ret, 0); - //printf("remove file success"); + // int ret = taosRemoveFile(fname); + // ASSERT_EQ(ret, 0); + // printf("remove file success"); } #ifndef OSFILE_PERFORMANCE_TEST @@ -178,18 +402,109 @@ TEST(osTest, osFile) { #define TESTTIMES 1000 char *getRandomWord() { - static char words[][MAX_WORD_LENGTH] = { - "Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit", - "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore", "magna", - "aliqua", "Ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco", - "Why", "do", "programmers", "prefer", "using", "dark", "mode?", "Because", "light", "attracts", - "bugs", "and", "they", "want", "to", "code", "in", "peace,", "like", "a", "ninja", "in", "the", "shadows." - "aliqua", "Ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco", - "laboris", "nisi", "ut", "aliquip", "ex", "ea", "commodo", "consequat", "Duis", "aute", "irure", - "dolor", "in", "reprehenderit", "in", "voluptate", "velit", "esse", "cillum", "dolore", "eu", - "fugiat", "nulla", "pariatur", "Excepteur", "sint", "occaecat", "cupidatat", "non", "proident", - "sunt", "in", "culpa", "qui", "officia", "deserunt", "mollit", "anim", "id", "est", "laborum" - }; + static char words[][MAX_WORD_LENGTH] = {"Lorem", + "ipsum", + "dolor", + "sit", + "amet", + "consectetur", + "adipiscing", + "elit", + "sed", + "do", + "eiusmod", + "tempor", + "incididunt", + "ut", + "labore", + "et", + "dolore", + "magna", + "aliqua", + "Ut", + "enim", + "ad", + "minim", + "veniam", + "quis", + "nostrud", + "exercitation", + "ullamco", + "Why", + "do", + "programmers", + "prefer", + "using", + "dark", + "mode?", + "Because", + "light", + "attracts", + "bugs", + "and", + "they", + "want", + "to", + "code", + "in", + "peace,", + "like", + "a", + "ninja", + "in", + "the", + "shadows." + "aliqua", + "Ut", + "enim", + "ad", + "minim", + "veniam", + "quis", + "nostrud", + "exercitation", + "ullamco", + "laboris", + "nisi", + "ut", + "aliquip", + "ex", + "ea", + "commodo", + "consequat", + "Duis", + "aute", + "irure", + "dolor", + "in", + "reprehenderit", + "in", + "voluptate", + "velit", + "esse", + "cillum", + "dolore", + "eu", + "fugiat", + "nulla", + "pariatur", + "Excepteur", + "sint", + "occaecat", + "cupidatat", + "non", + "proident", + "sunt", + "in", + "culpa", + "qui", + "officia", + "deserunt", + "mollit", + "anim", + "id", + "est", + "laborum"}; return words[taosRand() % MAX_WORDS]; } @@ -197,7 +512,7 @@ char *getRandomWord() { int64_t fillBufferWithRandomWords(char *buffer, int64_t maxBufferSize) { int64_t len = 0; while (len < maxBufferSize) { - char * word = getRandomWord(); + char *word = getRandomWord(); size_t wordLen = strlen(word); if (len + wordLen + 1 < maxBufferSize) { @@ -246,11 +561,11 @@ TEST(osTest, osFilePerformance) { int64_t OpenForWriteCloseFileCost; int64_t OpenForReadCloseFileCost; - char * buffer; - char * writeBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE); - char * readBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE); + char *buffer; + char *writeBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE); + char *readBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE); int64_t size = fillBufferWithRandomWords(writeBuffer, MAX_TEST_FILE_SIZE); - char * fname = "./osFilePerformanceTest.txt"; + char *fname = "./osFilePerformanceTest.txt"; TdFilePtr pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); ASSERT_NE(pOutFD, nullptr); @@ -342,6 +657,6 @@ TEST(osTest, osFilePerformance) { (void)printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost); } -#endif // OSFILE_PERFORMANCE_TEST +#endif // OSFILE_PERFORMANCE_TEST #pragma GCC diagnostic pop diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 20964c86bc..1f70c1abba 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -110,7 +110,7 @@ TEST(osThreadTests, thread) { taosMsleep(300); (void)taosThreadCancel(tid1); - + reti = taosThreadCreate(&tid2, NULL, funcPtr501, NULL); EXPECT_EQ(reti, 0); taosMsleep(1000); @@ -498,4 +498,42 @@ TEST(osThreadTests, spinlock) { TEST(osThreadTests, others) { taosThreadTestCancel(); taosThreadClear(NULL); + + TdThread tid1 = {0}; + TdThread tid2 = {0}; + TdThread *thread = &tid1; + taosResetPthread(NULL); + taosResetPthread(thread); + + bool retb = taosComparePthread(tid1, tid2); + EXPECT_TRUE(retb); + + int32_t ret32 = taosGetAppName(NULL, NULL); + EXPECT_NE(ret32, 0); + + char name[32] = {0}; + int32_t pid; + ret32 = taosGetPIdByName(name, NULL); + EXPECT_NE(ret32, 0); + ret32 = taosGetPIdByName(NULL, &pid); + EXPECT_NE(ret32, 0); + + ret32 = tsem_timewait(NULL, 124); + EXPECT_NE(ret32, 0); + ret32 = tsem_wait(NULL); + EXPECT_NE(ret32, 0); + ret32 = tsem2_init(NULL, 0, 0); + EXPECT_NE(ret32, 0); + ret32 = tsem_post(NULL); + EXPECT_NE(ret32, 0); + ret32 = tsem_destroy(NULL); + EXPECT_NE(ret32, 0); + ret32 = tsem2_post(NULL); + EXPECT_NE(ret32, 0); + ret32 = tsem2_destroy(NULL); + EXPECT_NE(ret32, 0); + ret32 = tsem2_wait(NULL); + EXPECT_NE(ret32, 0); + ret32 = tsem2_timewait(NULL, 128); + EXPECT_NE(ret32, 0); } \ No newline at end of file From 3c4fd115467e482f072b9f1c4768eec89e0a2e2b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 16:07:08 +0800 Subject: [PATCH 61/71] fix: add taosdump dump in out case --- .../taosdump/native/taosdumpCommandline.py | 74 +++++++++---------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index e93d938dac..f475d0eb63 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -14,6 +14,7 @@ import os import json import frame +import frame.eos import frame.etool from frame.log import * from frame.cases import * @@ -28,23 +29,19 @@ class TDTestCase(TBase): test taosdump support commandline arguments """ - def exec(self, command): - tdLog.info(command) - return os.system(command) - def clearPath(self, path): os.system("rm -rf %s/*" % path) def findPrograme(self): # taosdump - taosdump = etool.taosDumpFile() + taosdump = frame.etool.taosDumpFile() if taosdump == "": tdLog.exit("taosdump not found!") else: tdLog.info("taosdump found in %s" % taosdump) # taosBenchmark - benchmark = etool.benchMarkFile() + benchmark = frame.etool.benchMarkFile() if benchmark == "": tdLog.exit("benchmark not found!") else: @@ -60,7 +57,7 @@ class TDTestCase(TBase): return taosdump, benchmark,tmpdir - def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval=False): + def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval = True): # # check insert result # @@ -73,9 +70,9 @@ class TDTestCase(TBase): else: db = newdb - stb = data["databases"][0]["super_tables"][0]["name"] - child_count = data["databases"][0]["super_tables"][0]["childtable_count"] - insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] + stb = data["databases"][0]["super_tables"][0]["name"] + child_count = data["databases"][0]["super_tables"][0]["childtable_count"] + insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"] tdLog.info(f"get json: db={db} stb={stb} child_count={child_count} insert_rows={insert_rows} \n") @@ -91,12 +88,6 @@ class TDTestCase(TBase): tdSql.query(sql) tdSql.checkRows(0) - def testBenchmarkJson(self, benchmark, jsonFile, options="", checkInterval=False): - # exe insert - cmd = f"{benchmark} {options} -f {jsonFile}" - self.exec(cmd) - self.checkCorrectWithJson(jsonFile) - def insertData(self, json): # insert super table db, stb, child_count, insert_rows = self.insertBenchJson(json) @@ -115,14 +106,6 @@ class TDTestCase(TBase): return db, stb, child_count, insert_rows - def dumpOut(self, taosdump, db , outdir): - # dump out - self.exec(f"{taosdump} -D {db} -o {outdir}") - - def dumpIn(self, taosdump, db, newdb, indir): - # dump in - self.exec(f'{taosdump} -W "{db}={newdb}" -i {indir}') - def checkSame(self, db, newdb, stb, aggfun): # sum pk db sql = f"select {aggfun} from {db}.{stb}" @@ -145,14 +128,34 @@ class TDTestCase(TBase): # compare sum(pk) stb = "meters" - self.checkSame(db, newdb, stb, "sum(pk)") - self.checkSame(db, newdb, stb, "sum(usi)") + self.checkSame(db, newdb, stb, "sum(fc)") + self.checkSame(db, newdb, stb, "avg(dc)") + self.checkSame(db, newdb, stb, "sum(ti)") + self.checkSame(db, newdb, stb, "sum(si)") self.checkSame(db, newdb, stb, "sum(ic)") + self.checkSame(db, newdb, stb, "avg(bi)") + self.checkSame(db, newdb, stb, "sum(uti)") + self.checkSame(db, newdb, stb, "sum(usi)") + self.checkSame(db, newdb, stb, "sum(ui)") + self.checkSame(db, newdb, stb, "avg(ubi)") # check normal table self.checkSame(db, newdb, "ntb", "sum(c1)") - + # with Native Rest and WebSocket + def dumpInOutMode(self, mode, db, json, tmpdir): + # dump out + self.clearPath(tmpdir) + self.taosdump(f"{mode} -D {db} -o {tmpdir}") + + # dump in + newdb = "new" + db + self.taosdump(f"{mode} -W '{db}={newdb}' -i {tmpdir}") + + # check same + self.verifyResult(db, newdb, json) + + # basic commandline def basicCommandLine(self, tmpdir): #command and check result @@ -187,7 +190,7 @@ class TDTestCase(TBase): # check except def checkExcept(self, command): try: - code = self.exec(command) + code = frame.eos.exe(command, show = True) if code == 0: tdLog.exit(f"Failed, not report error cmd:{command}") else: @@ -220,7 +223,7 @@ class TDTestCase(TBase): json = "./tools/taosdump/native/json/insertFullType.json" # insert data with taosBenchmark - db, stb, child_count, insert_rows = self.insertData(json) + db, stb, childCount, insertRows = self.insertData(json) newdb = "new" + db # basic commandline @@ -229,15 +232,10 @@ class TDTestCase(TBase): # except commandline self.exceptCommandLine(taosdump, db, stb, tmpdir) - # dump out - #self.dumpOut(taosdump, db, tmpdir) - - # dump in - #self.dumpIn(taosdump, db, newdb, tmpdir) - - # verify db - #self.verifyResult(db, newdb, json) - + # dumpInOut + modes = ["", "-R" , "--cloud=http://localhost:6041"] + for mode in modes: + self.dumpInOutMode(mode, db , json, tmpdir) def stop(self): tdSql.close() From 4c3e6fbd19e188fc2d4d8c81393bd95ce5c3b9dd Mon Sep 17 00:00:00 2001 From: Simon Guan Date: Tue, 25 Feb 2025 16:39:05 +0800 Subject: [PATCH 62/71] fix: os unitest --- source/os/test/CMakeLists.txt | 84 ++++++++++++++++------------------- 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index a26fc88916..4b7cb3f7b9 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -15,36 +15,47 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) if(TD_LINUX) -add_executable(osAtomicTests "osAtomicTests.cpp") -target_link_libraries(osAtomicTests os util gtest_main) -add_test( - NAME osAtomicTests - COMMAND osAtomicTests -) -endif() + add_executable(osAtomicTests "osAtomicTests.cpp") + target_link_libraries(osAtomicTests os util gtest_main) + add_test( + NAME osAtomicTests + COMMAND osAtomicTests + ) -if(TD_LINUX) -add_executable(osDirTests "osDirTests.cpp") -target_link_libraries(osDirTests os util gtest_main) -add_test( - NAME osDirTests - COMMAND osDirTests -) -add_executable(osFileTests "osFileTests.cpp") -target_link_libraries(osFileTests os util gtest_main) -add_test( - NAME osFileTests - COMMAND osFileTests -) -endif() + add_executable(osDirTests "osDirTests.cpp") + target_link_libraries(osDirTests os util gtest_main) + add_test( + NAME osDirTests + COMMAND osDirTests + ) -if(TD_LINUX) -add_executable(osEnvTests "osEnvTests.cpp") -target_link_libraries(osEnvTests os util gtest_main) -add_test( - NAME osEnvTests - COMMAND osEnvTests -) + add_executable(osFileTests "osFileTests.cpp") + target_link_libraries(osFileTests os util gtest_main) + add_test( + NAME osFileTests + COMMAND osFileTests + ) + + add_executable(osEnvTests "osEnvTests.cpp") + target_link_libraries(osEnvTests os util gtest_main) + add_test( + NAME osEnvTests + COMMAND osEnvTests + ) + + add_executable(osTests "osTests.cpp") + target_link_libraries(osTests os util gtest_main) + add_test( + NAME osTests + COMMAND osTests + ) + + add_executable(osThreadTests "osThreadTests.cpp") + target_link_libraries(osThreadTests os util gtest_main) + add_test( + NAME osThreadTests + COMMAND osThreadTests + ) endif() add_executable(osMathTests "osMathTests.cpp") @@ -75,13 +86,6 @@ add_test( COMMAND osStringTests ) -add_executable(osTests "osTests.cpp") -target_link_libraries(osTests os util gtest_main) -add_test( - NAME osTests - COMMAND osTests -) - add_executable(osSystemTests "osSystemTests.cpp") target_link_libraries(osSystemTests os util gtest_main) add_test( @@ -89,19 +93,9 @@ add_test( COMMAND osSystemTests ) -if(TD_LINUX) -add_executable(osThreadTests "osThreadTests.cpp") -target_link_libraries(osThreadTests os util gtest_main) -add_test( - NAME osThreadTests - COMMAND osThreadTests -) -endif() - add_executable(osTimeTests "osTimeTests.cpp") target_link_libraries(osTimeTests os util gtest_main) add_test( NAME osTimeTests COMMAND osTimeTests ) - From a2f64ab8588c8d007aef3ad1c3d676db035d882e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 16:55:10 +0800 Subject: [PATCH 63/71] fix: ws can not support varbinary and geometry --- .../taosdump/native/json/insertFullType.json | 8 +-- .../taosdump/native/json/insertOther.json | 70 +++++++++++++++++++ .../taosdump/native/taosdumpCommandline.py | 35 +++++++--- 3 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 tests/army/tools/taosdump/native/json/insertOther.json diff --git a/tests/army/tools/taosdump/native/json/insertFullType.json b/tests/army/tools/taosdump/native/json/insertFullType.json index 5bdecef72b..a121670c37 100644 --- a/tests/army/tools/taosdump/native/json/insertFullType.json +++ b/tests/army/tools/taosdump/native/json/insertFullType.json @@ -41,9 +41,7 @@ { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, { "type": "binary", "name": "bin", "len": 4}, { "type": "nchar", "name": "nch", "len": 8}, - { "type": "varbinary", "name": "vab", "len": 8}, - { "type": "varchar", "name": "vac", "len": 8}, - { "type": "geometry", "name": "geo", "len": 32} + { "type": "varchar", "name": "vac", "len": 8} ], "tags":[ { "type": "bool", "name": "tbc"}, @@ -59,9 +57,7 @@ { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, { "type": "binary", "name": "tbin", "len": 4}, { "type": "nchar", "name": "tnch", "len": 8}, - { "type": "varbinary", "name": "tvab", "len": 8}, - { "type": "varchar", "name": "tvac", "len": 8}, - { "type": "geometry", "name": "tgeo", "len": 32} + { "type": "varchar", "name": "tvac", "len": 8} ] } ] diff --git a/tests/army/tools/taosdump/native/json/insertOther.json b/tests/army/tools/taosdump/native/json/insertOther.json new file mode 100644 index 0000000000..bd6bb12bec --- /dev/null +++ b/tests/army/tools/taosdump/native/json/insertOther.json @@ -0,0 +1,70 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"testother", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 2, + "insert_rows":100, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} + ], + "tags":[ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} + ] + } + ] + } + ] +} diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index f475d0eb63..c43ea0e9e7 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -121,7 +121,6 @@ class TDTestCase(TBase): else: tdLog.exit(f"{aggfun} source db:{sum1} import db:{sum2} not equal.") - def verifyResult(self, db, newdb, json): # compare with insert json self.checkCorrectWithJson(json, newdb) @@ -129,7 +128,6 @@ class TDTestCase(TBase): # compare sum(pk) stb = "meters" self.checkSame(db, newdb, stb, "sum(fc)") - self.checkSame(db, newdb, stb, "avg(dc)") self.checkSame(db, newdb, stb, "sum(ti)") self.checkSame(db, newdb, stb, "sum(si)") self.checkSame(db, newdb, stb, "sum(ic)") @@ -160,7 +158,7 @@ class TDTestCase(TBase): def basicCommandLine(self, tmpdir): #command and check result checkItems = [ - [f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], + [f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped"]], [f"-r result -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]], [f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], [f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], @@ -179,13 +177,14 @@ class TDTestCase(TBase): # executes for item in checkItems: + self.clearPath(tmpdir) command = item[0] results = item[1] rlist = self.taosdump(command) for result in results: self.checkListString(rlist, result) # clear tmp - self.clearPath(tmpdir) + # check except def checkExcept(self, command): @@ -224,19 +223,33 @@ class TDTestCase(TBase): # insert data with taosBenchmark db, stb, childCount, insertRows = self.insertData(json) - newdb = "new" + db - - # basic commandline - self.basicCommandLine(tmpdir) - - # except commandline - self.exceptCommandLine(taosdump, db, stb, tmpdir) # dumpInOut modes = ["", "-R" , "--cloud=http://localhost:6041"] for mode in modes: self.dumpInOutMode(mode, db , json, tmpdir) + tdLog.info("1. native rest ws dumpIn Out .......................... [Passed]") + + # basic commandline + self.basicCommandLine(tmpdir) + tdLog.info("2. basic command line .................................. [Passed]") + + # except commandline + self.exceptCommandLine(taosdump, db, stb, tmpdir) + tdLog.info("3. except command line ................................. [Passed]") + + # + # varbinary and geometry for native + # + json = "./tools/taosdump/native/json/insertOther.json" + # insert + db, stb, childCount, insertRows = self.insertData(json) + # dump in/out + self.dumpInOutMode("", db , json, tmpdir) + tdLog.info("4. native varbinary geometry ........................... [Passed]") + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From ba5fc20c203290582437107ab37355cdb723c9c2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Feb 2025 18:23:48 +0800 Subject: [PATCH 64/71] fix: add -B options to taosdumpCommandlinepy --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 45ea557c8e..1c0f550f5c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -207,7 +207,7 @@ ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeBool.py -B ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpRetry.py -B ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeTinyInt.py -B -,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py +,,n,army,python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py -B # # system test From 26f3adcf6ad9d4642d4163afd0c9f8dad6d407f4 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 25 Feb 2025 10:27:32 +0000 Subject: [PATCH 65/71] fix/TS-6028-check-column-name-fix-review --- source/dnode/mnode/impl/src/mndStb.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 5ada3e42e3..78f633acad 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1320,12 +1320,17 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - pHash = taosHashInit(3, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pHash = taosHashInit(createReq.numOfColumns + createReq.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_NO_LOCK); + if (pHash == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } void *pIter = NULL; for (int32_t i = 0; i < createReq.numOfColumns; ++i) { SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i); - if (taosHashPut(pHash, pField->name, sizeof(pField->name), pField->name, sizeof(pField->name)) != 0) { + if (taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0) != 0) { code = TSDB_CODE_TSC_DUP_COL_NAMES; goto _OVER; } @@ -1333,8 +1338,10 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { for (int32_t i = 0; i < createReq.numOfTags; ++i) { SField *pField = taosArrayGet(createReq.pTags, i); - if (taosHashPut(pHash, pField->name, sizeof(pField->name), pField->name, sizeof(pField->name)) != 0) { - code = TSDB_CODE_TSC_DUP_COL_NAMES; + if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) { + if (code == TSDB_CODE_DUP_KEY) { + code = TSDB_CODE_TSC_DUP_COL_NAMES; + } goto _OVER; } } From e2bbb11e72303cd078dd5b70e1c45872de595261 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 25 Feb 2025 19:10:28 +0800 Subject: [PATCH 66/71] fix:unit test --- source/client/test/clientTests.cpp | 56 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8ebbe62857..4a90e33bfa 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -152,13 +152,13 @@ void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t start for (int32_t i = 0; i < numOfRows; i += 20) { char sql[1024] = {0}; (void)sprintf(sql, - "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, - i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, - i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, + i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, + i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { (void)printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -170,14 +170,14 @@ void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t start for (int32_t i = 0; i < numOfRows; i += 20) { char sql[1024*50] = {0}; (void)sprintf(sql, - "insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, " - "%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, " - "'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')", - index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4, - pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9, - pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14, - i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18, - pVarchar, startTs + 19, i + 19, pVarchar); + "insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, " + "%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, " + "'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')", + index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4, + pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9, + pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14, + i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18, + pVarchar, startTs + 19, i + 19, pVarchar); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { (void)printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -309,8 +309,8 @@ int main(int argc, char** argv) { if (code != 0) { return code; } - - + + } numOfThreads = TMAX(numOfThreads, 1); @@ -1441,7 +1441,7 @@ TEST(clientCase, sub_tb_test) { int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); if (code != 0) { (void)printf("error occurs:%s\n", tmq_err2str(code)); - (void)tmq_consumer_close(tmq); + (void)tmq_consumer_close(tmq); taos_close(pConn); (void)fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); return; @@ -1523,29 +1523,29 @@ TEST(clientCase, timezone_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "drop database if exists db1"); + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_timezone"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); - pRes = taos_query(pConn, "create database db1"); + pRes = taos_query(pConn, "create database db_timezone"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); - pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)"); + pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)"); while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) { taosMsleep(2000); - pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)"); + pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)"); } ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); char sql[256] = {0}; - (void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:00', 1)"); + (void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:00', 1)"); pRes = taos_query(pConn, sql); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 17:00:00'"); + pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 17:00:00'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; @@ -1571,7 +1571,7 @@ TEST(clientCase, timezone_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 01:00:00'"); + TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 01:00:00'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; @@ -1588,7 +1588,7 @@ TEST(clientCase, timezone_Test) { taos_free_result(pRes); char sql[256] = {0}; - (void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:01', 1)"); + (void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:01', 1)"); pRes = taos_query(pConn, sql); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); @@ -1604,7 +1604,7 @@ TEST(clientCase, timezone_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 09:00:00'"); + TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 09:00:00'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; @@ -1620,7 +1620,7 @@ TEST(clientCase, timezone_Test) { taos_free_result(pRes); { - TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-17 01:00:01'"); + TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-17 01:00:01'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; From b1467e33bf06b2e216c92b1d83ae0ca8809556cd Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Tue, 25 Feb 2025 20:05:31 +0800 Subject: [PATCH 67/71] fix stmt2 unit test deadlock --- source/client/src/clientMain.c | 3 +++ source/client/src/clientStmt2.c | 13 +++++++------ source/client/test/stmt2Test.cpp | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 184f73c664..9a555682e6 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2272,7 +2272,10 @@ int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t c (void)atomic_add_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1); int code_s = taosStmt2AsyncBind(stmtAsyncBindThreadFunc, (void *)args); if (code_s != TSDB_CODE_SUCCESS) { + (void)taosThreadMutexLock(&(pStmt->asyncBindParam.mutex)); + (void)taosThreadCondSignal(&(pStmt->asyncBindParam.waitCond)); (void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1); + (void)taosThreadMutexUnlock(&(pStmt->asyncBindParam.mutex)); // terrno = TAOS_SYSTEM_ERROR(errno); } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 67066e1fdd..c06dbcb3fc 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1690,11 +1690,11 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { return pStmt->errCode; } - taosThreadMutexLock(&pStmt->asyncBindParam.mutex); - if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) { + (void)taosThreadMutexLock(&pStmt->asyncBindParam.mutex); + while (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) { (void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex); } - taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex); + (void)taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex); if (pStmt->sql.stbInterlaceMode) { STMT_ERR_RET(stmtAddBatch2(pStmt)); @@ -1796,11 +1796,12 @@ int stmtClose2(TAOS_STMT2* stmt) { pStmt->bindThreadInUse = false; } - taosThreadMutexLock(&pStmt->asyncBindParam.mutex); - if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) { + (void)taosThreadMutexLock(&pStmt->asyncBindParam.mutex); + while (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) { (void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex); } - taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex); + (void)taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex); + (void)taosThreadCondDestroy(&pStmt->queue.waitCond); (void)taosThreadMutexDestroy(&pStmt->queue.mutex); diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 72b2e6929c..638b964d10 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -15,6 +15,9 @@ #include #include +#include +#include +#include #include "clientInt.h" #include "geosWrapper.h" #include "osSemaphore.h" @@ -1662,7 +1665,7 @@ void stmtAsyncBindCb2(void* param, TAOS_RES* pRes, int code) { return; } -TEST(stmt2Case, async_order) { +void stmt2_async_test(std::atomic& stop_task) { int CTB_NUMS = 2; int ROW_NUMS = 2; int CYC_NUMS = 2; @@ -1868,5 +1871,27 @@ TEST(stmt2Case, async_order) { taosMemoryFree(tbs[i]); } taosMemoryFree(tbs); + stop_task = true; } + +TEST(stmt2Case, async_order) { + std::atomic stop_task(false); + std::thread t(stmt2_async_test, std::ref(stop_task)); + + // 等待 60 秒钟 + auto start_time = std::chrono::steady_clock::now(); + while (!stop_task) { + auto elapsed_time = std::chrono::steady_clock::now() - start_time; + if (std::chrono::duration_cast(elapsed_time).count() > 60) { + FAIL() << "Test[stmt2_async_test] timed out"; + t.detach(); + break; + } + std::this_thread::sleep_for(std::chrono::seconds(1)); // 每 1s 检查一次 + } + if (t.joinable()) { + t.join(); + } +} + #pragma GCC diagnostic pop From ebd2e657c6a743f169abcd41d65dc2d6785e1ed0 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 26 Feb 2025 01:30:33 +0000 Subject: [PATCH 68/71] fix/TS-6028-check-column-name-fix-review --- source/dnode/mnode/impl/src/mndStb.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 78f633acad..05b9826ae8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1326,12 +1326,13 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - void *pIter = NULL; for (int32_t i = 0; i < createReq.numOfColumns; ++i) { SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i); - if (taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0) != 0) { - code = TSDB_CODE_TSC_DUP_COL_NAMES; + if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) { + if (code == TSDB_CODE_DUP_KEY) { + code = TSDB_CODE_TSC_DUP_COL_NAMES; + } goto _OVER; } } From cdf0018b47ae22f1b90209a7bd056541d1d28f2f Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 26 Feb 2025 10:38:03 +0000 Subject: [PATCH 69/71] feat/TS-5805-force-assign-leader-update-doc --- docs/en/14-reference/03-taos-sql/20-keywords.md | 1 + docs/zh/08-operation/18-ha/02-replica2.md | 4 ++++ docs/zh/14-reference/03-taos-sql/20-keywords.md | 1 + 3 files changed, 6 insertions(+) diff --git a/docs/en/14-reference/03-taos-sql/20-keywords.md b/docs/en/14-reference/03-taos-sql/20-keywords.md index cd7d9e7a4b..682dcd3f84 100644 --- a/docs/en/14-reference/03-taos-sql/20-keywords.md +++ b/docs/en/14-reference/03-taos-sql/20-keywords.md @@ -37,6 +37,7 @@ The list of keywords is as follows: | ASOF | | | AT_ONCE | | | ATTACH | | +| ASSIGN | Version 3.3.6.0 and later | ### B diff --git a/docs/zh/08-operation/18-ha/02-replica2.md b/docs/zh/08-operation/18-ha/02-replica2.md index 7f3eb2fe5c..e238e3f4f4 100644 --- a/docs/zh/08-operation/18-ha/02-replica2.md +++ b/docs/zh/08-operation/18-ha/02-replica2.md @@ -69,9 +69,13 @@ alter database replica 2|1 | ------- | ------ | | 没有 Vnode 发生故障： Arbitrator 故障（Mnode 宕机节点超过一个，导致 Mnode 无法选主）| **持续提供服务** | | 仅一个 Vnode 故障：VGroup 已经达成同步后，某一个 Vnode 才发生故障的 | **持续提供服务** | +| 仅一个 Vnode 故障：2个 Vnode 同时故障，故障前 VGroup 达成同步，但是只有一个 Vnode 从故障中恢复服务，另一个 Vnode 服务故障 | **通过下面的命令，强制指定leader, 继续提供服务** | | 仅一个 Vnode 故障：离线 Vnode 启动后，VGroup 未达成同步前，另一个 Vnode 服务故障的 | **无法提供服务** | | 两个 Vnode 都发生故障 | **无法提供服务** | +```sql +ASSIGN LEADER FORCE; +``` ## 常见问题 diff --git a/docs/zh/14-reference/03-taos-sql/20-keywords.md b/docs/zh/14-reference/03-taos-sql/20-keywords.md index 1e8fabf571..8d5a538dd5 100644 --- a/docs/zh/14-reference/03-taos-sql/20-keywords.md +++ b/docs/zh/14-reference/03-taos-sql/20-keywords.md @@ -38,6 +38,7 @@ description: TDengine 保留关键字的详细列表 | AT_ONCE | | | ATTACH | | | AUTO | 3.3.5.0 及后续版本 | +| ASSIGN | 3.3.6.0 及后续版本 | ### B |关键字|说明| From ec3bb5b742f8a569c026ee8612c448e3f05c8493 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 27 Feb 2025 10:15:38 +0800 Subject: [PATCH 70/71] feat:[TS-5776]process return value --- source/libs/parser/src/parInsertUtil.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 1931f0803d..eab9dda5bf 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -1079,11 +1079,14 @@ end: } int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data) { - transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion); + int code = transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion); + if (code != 0){ + return code; + } SVgroupDataCxt* pVgCxt = NULL; void** pp = taosHashGet(pVgroupHash, &pTableMeta->vgId, sizeof(pTableMeta->vgId)); if (NULL == pp) { - int code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt); + code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt); if (code != 0){ return code; } From 5635b285cd6be1829c51b524b8a1743bef2b6e82 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 27 Feb 2025 03:17:21 +0000 Subject: [PATCH 71/71] feat/TS-5805-force-assign-leader-fix-check --- source/dnode/mnode/impl/src/mndArbGroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 48c7dace16..af56283c96 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -1431,7 +1431,7 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char strCheckSyncCode[100] = {0}; char bufUpdateTime[40] = {0}; (void)formatTimestamp(bufUpdateTime, pGroup->updateTimeMs, TSDB_TIME_PRECISION_MILLI); - tsnprintf(strCheckSyncCode, 100, "%s(%s)", tstrerror(pGroup->code), bufUpdateTime); + (void)tsnprintf(strCheckSyncCode, 100, "%s(%s)", tstrerror(pGroup->code), bufUpdateTime); char checkSyncCode[100 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(checkSyncCode, strCheckSyncCode, 100 + VARSTR_HEADER_SIZE);