From 90fc572394bb37776cee92e08d188b96319560cc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 7 Nov 2022 15:32:48 +0800 Subject: [PATCH 01/56] avoid epset leak --- source/libs/scheduler/src/schRemote.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 47de2528fa..17dc3d588f 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -425,6 +425,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { _return: taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -438,6 +439,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { code); if (pMsg) { taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); } return TSDB_CODE_SUCCESS; } @@ -492,6 +494,7 @@ _return: tFreeSSchedulerHbRsp(&rsp); taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); SCH_RET(code); } From 495bfe932a7412c4587b32e890aebcb41b84c797 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 18:56:44 +0800 Subject: [PATCH 02/56] fix invalid free --- source/libs/scheduler/src/schTask.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index dbdccff302..7e5b3faedb 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -439,6 +439,8 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 code = schDoTaskRedirect(pJob, pTask, pData, rspCode); taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(code); @@ -446,6 +448,8 @@ _return: taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -942,7 +946,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &qwMsg, explainRes)); + pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -1115,7 +1119,7 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &pRsp, explainRes)); + pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); From 7b30a6ee23f9572a12f018cb6c1b54a6e220a99b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 20:45:11 +0800 Subject: [PATCH 03/56] free unfinished ahandle while quit --- include/libs/qcom/query.h | 2 ++ include/libs/transport/trpc.h | 4 +++ source/client/src/clientEnv.c | 3 ++- source/libs/qcom/src/queryUtil.c | 6 +++++ source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 31 +++++++++++++++--------- 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 651b379851..c6f7b5cf2e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -199,6 +199,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +void destroyAhandle(void* ahandle); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 53edcd4fee..8a0cccc71f 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -72,6 +72,7 @@ typedef struct SRpcMsg { typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; @@ -97,6 +98,9 @@ typedef struct SRpcInit { // set up timeout for particular msg RpcTfp tfp; + // destroy client ahandle; + RpcDfp dfp; + void *parent; } SRpcInit; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb25a595af..bb93e4d934 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -140,12 +140,13 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.rfp = clientRpcRfp; - // rpcInit.tfp = clientRpcTfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cf064881c2..6eadf80e3d 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -146,6 +146,12 @@ void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { } taosMemoryFreeClear(pMsgBody); } +void destroyAhandle(void *ahandle) { + SMsgSendInfo *pSendInfo = ahandle; + if (pSendInfo == NULL) return; + + destroySendMsgInfo(pSendInfo); +} int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index b9167501e2..3b7182f983 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -53,6 +53,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d3db4879d1..feb55985de 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -53,6 +53,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 21444018cd..42dc65abea 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -143,6 +143,7 @@ static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn); static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); +static void cliReleaseUnfinishedMsg(SCliConn* conn); // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); @@ -163,17 +164,6 @@ static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); -static void cliReleaseUnfinishedMsg(SCliConn* conn) { - for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { - SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); - if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { - if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { - conn->ctx.freeFunc(msg->ctx->ahandle); - } - } - destroyCmsg(msg); - } -} #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -266,6 +256,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { static void* cliWorkThread(void* arg); +static void cliReleaseUnfinishedMsg(SCliConn* conn) { + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { + SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); + if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { + if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { + conn->ctx.freeFunc(msg->ctx->ahandle); + continue; + } + if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); + pTransInst->destroyFp(msg->ctx->ahandle); + } + } + destroyCmsg(msg); + } +} bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pCliMsg = NULL; From 1ed333140b8ffd638d0a228b46c31f4e95c2acef Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 22:09:58 +0800 Subject: [PATCH 04/56] fix invalid free --- source/libs/transport/src/transCli.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 42dc65abea..bae9295749 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -265,9 +265,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { conn->ctx.freeFunc(msg->ctx->ahandle); - continue; - } - if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + } else if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); pTransInst->destroyFp(msg->ctx->ahandle); } From 97fb25372ef66344bac952c253ee40281dd2cc7b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 9 Nov 2022 12:58:44 +0800 Subject: [PATCH 05/56] fix(ci):modify ci test --- tests/script/tsim/stream/basic1.sim | 428 ++++++++++++++++------------ 1 file changed, 253 insertions(+), 175 deletions(-) diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index b20e2e3592..bb6604687c 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -23,443 +23,516 @@ sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); -sleep 1000 + +$loop_count = 0 + +loop0: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; -if $rows != 4 then - print ======$rows +$loop_count = $loop_count + 1 +if $loop_count == 20 then return -1 endi +if $rows != 4 then + print =====rows=$rows + goto loop0 +endi + # row 0 if $data01 != 2 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop0 endi if $data02 != 2 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop0 endi if $data03 != 5 then - print ======$data03 - return -1 + print =====data03=$data03 + goto loop0 endi if $data04 != 2 then - print ======$data04 - return -1 + print =====data04=$data04 + goto loop0 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop0 endi # row 1 if $data11 != 1 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop0 endi if $data12 != 1 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop0 endi if $data13 != 2 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop0 endi if $data14 != 2 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop0 endi if $data15 != 3 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop0 endi # row 2 if $data21 != 1 then - print ======$data21 + print =====data21=$data21 return -1 endi if $data22 != 1 then - print ======$data22 + print =====data22=$data22 return -1 endi if $data23 != 3 then - print ======$data23 + print =====data23=$data23 return -1 endi if $data24 != 2 then - print ======$data24 + print =====data24=$data24 return -1 endi if $data25 != 3 then - print ======$data25 + print =====data25=$data25 return -1 endi # row 3 if $data31 != 1 then - print ======$data31 + print =====data31=$data31 return -1 endi if $data32 != 1 then - print ======$data32 + print =====data32=$data32 return -1 endi if $data33 != 4 then - print ======$data33 + print =====data33=$data33 return -1 endi if $data34 != 2 then - print ======$data34 + print =====data34=$data34 return -1 endi if $data35 != 3 then - print ======$data35 + print =====data35=$data35 return -1 endi sql insert into t1 values(1648791223001,12,14,13,11.1); -sleep 500 + +$loop_count = 0 +loop1: +sleep 200 + sql select * from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + print count(*) , count(d) , sum(a) , max(b) , min(c) print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 if $rows != 4 then print ======$rows - return -1 + goto loop1 endi # row 0 if $data01 != 2 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop1 endi if $data02 != 2 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop1 endi if $data03 != 5 then - print ======$data03 - return -1 + print =====data03=$data03 + goto loop1 endi if $data04 != 2 then - print ======$data04 - return -1 + print =====data04=$data04 + goto loop1 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop1 endi # row 1 if $data11 != 1 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop1 endi if $data12 != 1 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop1 endi if $data13 != 12 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop1 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop1 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop1 endi # row 2 if $data21 != 1 then - print ======$data21 + print =====data21=$data21 return -1 endi if $data22 != 1 then - print ======$data22 + print =====data22=$data22 return -1 endi if $data23 != 3 then - print ======$data23 + print =====data23=$data23 return -1 endi if $data24 != 2 then - print ======$data24 + print =====data24=$data24 return -1 endi if $data25 != 3 then - print ======$data25 + print =====data25=$data25 return -1 endi # row 3 if $data31 != 1 then - print ======$data31 + print =====data31=$data31 return -1 endi if $data32 != 1 then - print ======$data32 + print =====data32=$data32 return -1 endi if $data33 != 4 then - print ======$data33 + print =====data33=$data33 return -1 endi if $data34 != 2 then - print ======$data34 + print =====data34=$data34 return -1 endi if $data35 != 3 then - print ======$data35 + print =====data35=$data35 return -1 endi sql insert into t1 values(1648791223002,12,14,13,11.1); -sleep 100 + +$loop_count = 0 +loop2: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 2 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop2 endi if $data12 != 2 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop2 endi if $data13 != 24 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop2 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop2 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop2 endi sql insert into t1 values(1648791223003,12,14,13,11.1); -sleep 100 + +$loop_count = 0 +loop3: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 3 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop3 endi if $data12 != 3 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop3 endi if $data13 != 36 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop3 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop3 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop3 endi sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); -sleep 100 + +$loop_count = 0 +loop4: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 3 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop4 endi if $data12 != 3 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop4 endi if $data13 != 6 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop4 endi if $data14 != 3 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop4 endi if $data15 != 1 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop4 endi sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); -sleep 100 + +$loop_count = 0 +loop5: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 2 if $data21 != 2 then - print ======$data21 - return -1 + print =====data21=$data21 + goto loop5 endi if $data22 != 2 then - print ======$data22 - return -1 + print =====data22=$data22 + goto loop5 endi if $data23 != 6 then - print ======$data23 - return -1 + print =====data23=$data23 + goto loop5 endi if $data24 != 2 then - print ======$data24 - return -1 + print =====data24=$data24 + goto loop5 endi if $data25 != 3 then - print ======$data25 - return -1 + print =====data25=$data25 + goto loop5 endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); -sleep 100 + +$loop_count = 0 +loop6: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 0 if $data01 != 4 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop6 endi if $data02 != 4 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop6 endi if $data03 != 50 then - print ======$data03 != 50 - return -1 + print =====data03=$data03 != 50 + goto loop6 endi if $data04 != 20 then - print ======$data04 != 20 - return -1 + print =====data04=$data04 != 20 + goto loop6 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop6 endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); -sleep 100 + +$loop_count = 0 +loop7: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 4 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop7 endi if $data12 != 4 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop7 endi if $data13 != 46 then - print ======$data13 != 46 - return -1 + print =====data13=$data13 != 46 + goto loop7 endi if $data14 != 20 then - print ======$data14 != 20 - return -1 + print =====data14=$data14 != 20 + goto loop7 endi if $data15 != 1 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop7 endi # row 2 if $data21 != 4 then - print ======$data21 - return -1 + print =====data21=$data21 + goto loop7 endi if $data22 != 4 then - print ======$data22 - return -1 + print =====data22=$data22 + goto loop7 endi if $data23 != 15 then - print ======$data23 - return -1 + print =====data23=$data23 + goto loop7 endi if $data24 != 4 then - print ======$data24 - return -1 + print =====data24=$data24 + goto loop7 endi if $data25 != 3 then - print ======$data25 - return -1 + print =====data25=$data25 + goto loop7 endi sql create database test2 vgroups 1; @@ -479,11 +552,11 @@ sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2 $loop_count = 0 -loop0: -sleep 300 +loop8: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -491,7 +564,7 @@ sql select * from streamt; if $rows != 4 then print =====rows=$rows - goto loop0 + goto loop8 endi sql insert into t1 values(1648791213000,5,5,5,5.0) t2 values(1648791213000,6,6,6,6.0) t5 values(1648791213000,7,7,7,7.0); @@ -499,11 +572,11 @@ sql insert into t1 values(1648791213000,5,5,5,5.0) t2 values(1648791213000,6,6,6 $loop_count = 0 -loop1: -sleep 300 +loop9: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -511,51 +584,51 @@ sql select * from streamt order by c4 desc; if $rows != 5 then print =====rows=$rows - goto loop1 + goto loop9 endi # row 0 if $data01 != 1 then print =====data01=$data01 - goto loop1 + goto loop9 endi if $data02 != 7 then print =====data02=$data02 - goto loop1 + goto loop9 endi # row 1 if $data11 != 1 then print =====data11=$data11 - goto loop1 + goto loop9 endi if $data12 != 6 then print =====data12=$data12 - goto loop1 + goto loop9 endi # row 2 if $data21 != 1 then print =====data21=$data21 - goto loop1 + goto loop9 endi if $data22 != 5 then print =====data22=$data22 - goto loop1 + goto loop9 endi sql insert into t1 values(1648791213000,8,8,8,8.0); $loop_count = 0 -loop2: -sleep 300 +loop10: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -564,28 +637,29 @@ sql select * from streamt order by c4 desc; # row 0 if $data01 != 1 then print =====data01=$data01 - goto loop2 + goto loop10 endi if $data02 != 8 then print =====data02=$data02 - goto loop2 + goto loop10 endi $loop_count = 0 -loop3: -sleep 300 +loop11: +sleep 200 + +sql select count(*) from streamt3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi -sql select count(*) from streamt3; # row 0 if $data00 != 5 then print =====data00=$data00 - goto loop3 + goto loop11 endi #max,min selectivity @@ -601,25 +675,26 @@ sql insert into ts1 values(1648791222001,2,2,3); sleep 50 $loop_count = 0 -loop3: +loop12: +sleep 200 + sql select * from streamtST3; -sleep 300 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi # row 0 if $data02 != 1 then print =====data02=$data02 - goto loop3 + goto loop12 endi # row 1 if $data12 != 2 then print =====data12=$data12 - goto loop3 + goto loop12 endi @@ -629,19 +704,22 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); +$loop_count = 0 +loop13: sleep 200 + sql select * from streamt4; # row 0 if $rows != 0 then print =====rows=$rows - return -1 + goto loop13 endi sql insert into t1 values(1648791213000,6,2,3,1.0); $loop_count = 0 -loop4: +loop14: sleep 200 sql select * from streamt4; @@ -652,13 +730,13 @@ endi if $data01 != 1 then print =====data01=$data01 - goto loop4 + goto loop14 endi sql insert into t1 values(1648791213000,2,2,3,1.0); $loop_count = 0 -loop5: +loop15: sleep 200 sql select * from streamt4; @@ -669,7 +747,7 @@ endi if $rows != 0 then print =====rows=$rows - goto loop5 + goto loop15 endi From b48752686e489a86d2597b91e4d8dcde16616efb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 13:45:46 +0800 Subject: [PATCH 06/56] refactor: do some internal refactor. --- include/common/tdatablock.h | 2 +- source/client/src/clientMsgHandler.c | 3 +- source/common/src/tdatablock.c | 31 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 3 +- source/dnode/mnode/impl/src/mndShow.c | 3 +- source/dnode/vnode/src/tq/tqExec.c | 4 +- source/libs/command/src/command.c | 3 +- source/libs/command/src/explain.c | 3 +- source/libs/executor/inc/executorimpl.h | 8 +- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/executor/src/executor.c | 21 + source/libs/executor/src/executorimpl.c | 628 +----------------- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/joinoperator.c | 7 +- source/libs/executor/src/projectoperator.c | 5 +- source/libs/executor/src/scanoperator.c | 17 +- source/libs/executor/src/sortoperator.c | 6 +- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 22 +- source/libs/stream/src/streamDispatch.c | 6 +- 21 files changed, 87 insertions(+), 697 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index fdbb2c5123..502ba10d33 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -244,7 +244,7 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e17e7e79c6..591d469d7c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -442,8 +442,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d0121aa15..536cbed33e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2197,7 +2197,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { return rname.ctbShortName; } -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { + int32_t dataLen = 0; + // todo extract method int32_t* version = (int32_t*)data; *version = 1; @@ -2238,7 +2240,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = blockDataGetSerialMetaSize(numOfCols); + dataLen = blockDataGetSerialMetaSize(numOfCols); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -2255,26 +2257,23 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ } data += metaSize; - (*dataLen) += metaSize; + dataLen += metaSize; - if (needCompress) { - colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); - data += colSizes[col]; - (*dataLen) += colSizes[col]; - } else { - colSizes[col] = colDataGetLength(pColRes, numOfRows); - (*dataLen) += colSizes[col]; - memmove(data, pColRes->pData, colSizes[col]); - data += colSizes[col]; - } + colSizes[col] = colDataGetLength(pColRes, numOfRows); + dataLen += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); } - *actualLen = *dataLen; + *actualLen = dataLen; *groupId = pBlock->info.groupId; - ASSERT(*dataLen > 0); - uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols); + ASSERT(dataLen > 0); + + uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols); + + return dataLen; } const char* blockDecode(SSDataBlock* pBlock, const char* pData) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 85a09b79fd..8a8561161b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -307,8 +307,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, pStart, numOfCols); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 637a3818e6..b0af98b933 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -303,8 +303,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index a6e8767a4d..48c14bc758 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -27,9 +27,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->completed = 1; pRetrieve->numOfRows = htonl(pBlock->info.rows); - // TODO enable compress - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 76c84b3be9..1c2d7e1f66 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -39,8 +39,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 80a524496c..915dc08c14 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1610,8 +1610,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - int32_t len = 0; - blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ced668cf37..77915624dd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -842,10 +842,8 @@ typedef struct SJoinOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) -void doDestroyExchangeOperatorInfo(void* param); - -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain); +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); @@ -881,6 +879,8 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); +void doDestroyExchangeOperatorInfo(void* param); + void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 76866eedb7..41fcc44f97 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -102,7 +102,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, NULL, destroyLastrowScanOperator, NULL); + createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyLastrowScanOperator, NULL); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index bc4ab9c468..3d9757c96f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1cf01c8661..1aa9a3c613 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1106,3 +1106,24 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } return 0; } + +void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; + assert(pMsg->info.ahandle != NULL); + + SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; + + if (pMsg->contLen > 0) { + buf.pData = taosMemoryCalloc(1, pMsg->contLen); + if (buf.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(buf.pData, pMsg->pCont, pMsg->contLen); + } + } + + pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); + rpcFreeCont(pMsg->pCont); + destroySendMsgInfo(pSendInfo); +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7d662f8784..ca1d4cccab 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -32,7 +32,6 @@ #include "index.h" #include "query.h" #include "tcompare.h" -#include "tcompression.h" #include "thash.h" #include "ttypes.h" #include "vnode.h" @@ -91,7 +90,7 @@ static void destroySortOperatorInfo(void* param); static void destroyAggOperatorInfo(void* param); static void destroyIntervalOperatorInfo(void* param); -static void destroyExchangeOperatorInfo(void* param); + static void destroyOperatorInfo(SOperatorInfo* pOperator); @@ -109,8 +108,8 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, @@ -1652,596 +1651,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t return TSDB_CODE_SUCCESS; } -typedef struct SFetchRspHandleWrapper { - uint32_t exchangeId; - int32_t sourceIndex; -} SFetchRspHandleWrapper; - -int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { - SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; - - SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); - if (pExchangeInfo == NULL) { - qWarn("failed to acquire exchange operator, since it may have been released"); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; - } - - int32_t index = pWrapper->sourceIndex; - SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - - if (code == TSDB_CODE_SUCCESS) { - pSourceDataInfo->pRsp = pMsg->pData; - - SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->compLen = htonl(pRsp->compLen); - pRsp->numOfCols = htonl(pRsp->numOfCols); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - - ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows); - } else { - taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); - } - - pSourceDataInfo->status = EX_SOURCE_DATA_READY; - - tsem_post(&pExchangeInfo->ready); - taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - - return TSDB_CODE_SUCCESS; -} - -void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); - - SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; - - if (pMsg->contLen > 0) { - buf.pData = taosMemoryCalloc(1, pMsg->contLen); - if (buf.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - memcpy(buf.pData, pMsg->pCont, pMsg->contLen); - } - } - - pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); - rpcFreeCont(pMsg->pCont); - destroySendMsgInfo(pSendInfo); -} - -static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); - - ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); - - SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); - pWrapper->exchangeId = pExchangeInfo->self; - pWrapper->sourceIndex = sourceIndex; - - if (pSource->localExec) { - SDataBuf pBuf = {0}; - int32_t code = - (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, - pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); - loadRemoteDataCallback(pWrapper, &pBuf, code); - taosMemoryFree(pWrapper); - } else { - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - taosMemoryFree(pWrapper); - return pTaskInfo->code; - } - - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, sourceIndex, totalSources); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); - taosMemoryFree(pWrapper); - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return pTaskInfo->code; - } - - pMsgSendInfo->param = pWrapper; - pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = pSource->fetchMsgType; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); - } - - return TSDB_CODE_SUCCESS; -} - -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator) { - pInfo->totalRows += numOfRows; - pInfo->totalSize += dataLen; - pInfo->totalElapsed += (taosGetTimestampUs() - startTs); - pOperator->resultInfo.totalRows += numOfRows; -} - -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { - if (pColList == NULL) { // data from other sources - blockDataCleanup(pRes); - *pNextStart = (char*)blockDecode(pRes, pData); - } else { // extract data according to pColList - char* pStart = pData; - - int32_t numOfCols = htonl(*(int32_t*)pStart); - pStart += sizeof(int32_t); - - // todo refactor:extract method - SSysTableSchema* pSchema = (SSysTableSchema*)pStart; - for (int32_t i = 0; i < numOfCols; ++i) { - SSysTableSchema* p = (SSysTableSchema*)pStart; - - p->colId = htons(p->colId); - p->bytes = htonl(p->bytes); - pStart += sizeof(SSysTableSchema); - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); - blockDataAppendColInfo(pBlock, &idata); - } - - blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, pBlock->info.rows); - - // data from mnode - pRes->info.rows = pBlock->info.rows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - blockDataDestroy(pBlock); - } - - // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. - blockDataUpdateTsWindow(pRes, 0); - return TSDB_CODE_SUCCESS; -} - -static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int64_t el = taosGetTimestampUs() - startTs; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - - pLoadInfo->totalElapsed += el; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - - doSetOperatorCompleted(pOperator); - return NULL; -} - - -static int32_t getCompletedSources(const SArray* pArray) { - size_t total = taosArrayGetSize(pArray); - - int32_t completed = 0; - for (int32_t k = 0; k < total; ++k) { - SSourceDataInfo* p = taosArrayGet(pArray, k); - if (p->status == EX_SOURCE_DATA_EXHAUSTED) { - completed += 1; - } - } - - return completed; -} - -static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { - int32_t code = 0; - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); - int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); - if (completed == totalSources) { - setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); - return; - } - - while (1) { - tsem_wait(&pExchangeInfo->ready); - - for (int32_t i = 0; i < totalSources; ++i) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { - continue; - } - - if (pDataInfo->status != EX_SOURCE_DATA_READY) { - continue; - } - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - code = pDataInfo->code; - goto _error; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - - // todo - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); - taosMemoryFreeClear(pDataInfo->pRsp); - break; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - int32_t index = 0; - char* pStart = pRetrieveRsp->data; - while (index++ < pRetrieveRsp->numOfBlocks) { - SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - - taosArrayPush(pExchangeInfo->pResultBlockList, &pb); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); - - if (pRsp->completed == 1) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, - i + 1, totalSources); - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); - } - - taosMemoryFreeClear(pDataInfo->pRsp); - - if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { - pDataInfo->status = EX_SOURCE_DATA_NOT_READY; - code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - } - return; - } // end loop - - int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); - if (complete1 == totalSources) { - qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); - return; - } - } - -_error: - pTaskInfo->code = code; -} - -static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - // Asynchronously send all fetch requests to all sources. - for (int32_t i = 0; i < totalSources; ++i) { - int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return code; - } - } - - int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), - totalSources, (endTs - startTs) / 1000.0); - - pOperator->status = OP_RES_TO_RETURN; - pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); - tsem_post(&pExchangeInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - while (1) { - if (pExchangeInfo->current >= totalSources) { - setAllSourcesCompleted(pOperator, startTs); - return TSDB_CODE_SUCCESS; - } - - doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); - tsem_wait(&pExchangeInfo->ready); - - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); - pOperator->pTaskInfo->code = pDataInfo->code; - return pOperator->pTaskInfo->code; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pLoadInfo->totalRows); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - taosMemoryFreeClear(pDataInfo->pRsp); - continue; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - - char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); - - if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, - totalSources); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pLoadInfo->totalRows, pLoadInfo->totalSize); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); - pDataInfo->totalRows += pRetrieveRsp->numOfRows; - - taosMemoryFreeClear(pDataInfo->pRsp); - return TSDB_CODE_SUCCESS; - } -} - -static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - int64_t st = taosGetTimestampUs(); - - SExchangeInfo* pExchangeInfo = pOperator->info; - if (!pExchangeInfo->seqLoadData) { - int32_t code = prepareConcurrentlyLoad(pOperator); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - pExchangeInfo->openedTs = taosGetTimestampUs(); - } - - OPTR_SET_OPENED(pOperator); - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return TSDB_CODE_SUCCESS; -} - -static void freeBlock(void* pParam) { - SSDataBlock* pBlock = *(SSDataBlock**)pParam; - blockDataDestroy(pBlock); -} - -static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pOperator->status == OP_EXEC_DONE) { - qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - return NULL; - } - - size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); - if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { - pExchangeInfo->rspBlockIndex = 0; - taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); - if (pExchangeInfo->seqLoadData) { - seqLoadRemoteData(pOperator); - } else { - concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - } - - if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { - return NULL; - } - } - - // we have buffered retrieved datablock, return it directly - return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); -} - -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); - if (pBlock == NULL) { - return NULL; - } - - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; - if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); - if (status == PROJECT_RETRIEVE_CONTINUE) { - continue; - } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pBlock->info.rows; - pExchangeInfo->limitInfo.numOfOutputRows += rows; - - if (rows == 0) { - doSetOperatorCompleted(pOperator); - return NULL; - } else { - return pBlock; - } - } - } else { - return pBlock; - } - } -} - -static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { - pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); - if (pInfo->pSourceDataInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SSourceDataInfo dataInfo = {0}; - dataInfo.status = EX_SOURCE_DATA_NOT_READY; - dataInfo.taskId = id; - dataInfo.index = i; - SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); - if (pDs == NULL) { - taosArrayDestroy(pInfo->pSourceDataInfo); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { - size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); - - if (numOfSources == 0) { - qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); - return TSDB_CODE_INVALID_PARA; - } - - pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); - if (pInfo->pSources == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); - taosArrayPush(pInfo->pSources, pNode); - } - - initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); - pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); - - return initDataSource(numOfSources, pInfo, id); -} - -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tsem_init(&pInfo->ready, 0, 0); - pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); - - pInfo->seqLoadData = false; - pInfo->pTransporter = pTransporter; - - pOperator->name = "ExchangeOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - doDestroyExchangeOperatorInfo(pInfo); - } - - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); @@ -2960,7 +2369,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, NULL); + createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -3024,33 +2433,6 @@ void destroyFillOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void destroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - taosRemoveRef(exchangeObjRefPool, pExInfo->self); -} - -void freeSourceDataInfo(void* p) { - SSourceDataInfo* pInfo = (SSourceDataInfo*)p; - taosMemoryFreeClear(pInfo->pRsp); -} - -void doDestroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - - taosArrayDestroy(pExInfo->pSources); - taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); - - if (pExInfo->pResultBlockList != NULL) { - taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); - pExInfo->pResultBlockList = NULL; - } - - blockDataDestroy(pExInfo->pDummyBlock); - - tsem_destroy(&pExInfo->ready); - taosMemoryFreeClear(param); -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { @@ -3190,7 +2572,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 891eb6e7a4..cfcdd0ee64 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -446,7 +446,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -831,7 +831,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1111,7 +1111,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 45d76dce74..e7fe2bfc12 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -121,8 +121,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,13 +371,13 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) if (leftTs == rightTs) { mergeJoinJoinDownstreamTsRanges(pOperator, leftTs, pRes, &nrows); - } else if (asc && leftTs < rightTs || !asc && leftTs > rightTs) { + } else if ((asc && leftTs < rightTs) || (!asc && leftTs > rightTs)) { pJoinInfo->leftPos += 1; if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { continue; } - } else if (asc && leftTs > rightTs || !asc && leftTs < rightTs) { + } else if ((asc && leftTs > rightTs) || (!asc && leftTs < rightTs)) { pJoinInfo->rightPos += 1; if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { continue; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index b2865d15f0..58cc082653 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -104,7 +104,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -406,8 +406,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, - destroyIndefinitOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6d13048123..903843bcc4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -962,7 +962,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 @@ -993,7 +993,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1155,8 +1155,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; _error: @@ -2372,7 +2371,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; _end: @@ -2565,7 +2564,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pOperator->pTaskInfo = pTaskInfo; __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); return pOperator; @@ -4207,7 +4206,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; _error: @@ -4346,7 +4345,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); return pOperator; @@ -4861,7 +4860,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index af87c4b2eb..700612e0ab 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -67,7 +67,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -517,7 +517,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); @@ -781,7 +781,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 9f06e639b3..a6ac28829c 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1698,7 +1698,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7b6ed5b67d..6d8282ff5d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1785,7 +1785,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2565,7 +2565,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -2641,7 +2641,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2718,7 +2718,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -3403,7 +3403,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -4196,7 +4196,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, @@ -4342,7 +4342,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pOperator->name = "StreamSessionSemiAggOperator"; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); } @@ -4713,7 +4713,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->pTaskInfo = pTaskInfo; pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); @@ -4995,7 +4995,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = miaInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5307,7 +5307,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pOperator->info = pMergeIntervalInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5540,7 +5540,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index ad342edfa0..573a1ea31f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -118,8 +118,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -200,8 +199,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From dc67c5041a360e495e4c49f13b9e7f3dd502180b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 13:46:19 +0800 Subject: [PATCH 07/56] refactor: do some internal refactor. --- source/libs/executor/src/exchangeoperator.c | 643 ++++++++++++++++++++ 1 file changed, 643 insertions(+) create mode 100644 source/libs/executor/src/exchangeoperator.c diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c new file mode 100644 index 0000000000..cb3330e9ca --- /dev/null +++ b/source/libs/executor/src/exchangeoperator.c @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" +#include "tref.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "tsort.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef struct SFetchRspHandleWrapper { + uint32_t exchangeId; + int32_t sourceIndex; +} SFetchRspHandleWrapper; + +static void destroyExchangeOperatorInfo(void* param); +static void freeBlock(void* pParam); +static void freeSourceDataInfo(void* param); +static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs); + +static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); +static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex); +static int32_t getCompletedSources(const SArray* pArray); +static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); +static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); + +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { + int32_t code = 0; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); + int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (completed == totalSources) { + setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); + return; + } + + while (1) { + tsem_wait(&pExchangeInfo->ready); + + for (int32_t i = 0; i < totalSources; ++i) { + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); + if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { + continue; + } + + if (pDataInfo->status != EX_SOURCE_DATA_READY) { + continue; + } + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + code = pDataInfo->code; + goto _error; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); + + // todo + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, + pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); + taosMemoryFreeClear(pDataInfo->pRsp); + break; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while (index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); + + if (pRsp->completed == 1) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + ", total:%.2f Kb, try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + i + 1, totalSources); + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); + } + + taosMemoryFreeClear(pDataInfo->pRsp); + + if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { + pDataInfo->status = EX_SOURCE_DATA_NOT_READY; + code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + } + return; + } // end loop + + int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (complete1 == totalSources) { + qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); + return; + } + } + + _error: + pTaskInfo->code = code; +} + +static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pOperator->status == OP_EXEC_DONE) { + qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + return NULL; + } + + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } + } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); +} + +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); + if (pBlock == NULL) { + return NULL; + } + + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; + if (hasLimitOffsetInfo(pLimitInfo)) { + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { + size_t rows = pBlock->info.rows; + pExchangeInfo->limitInfo.numOfOutputRows += rows; + + if (rows == 0) { + doSetOperatorCompleted(pOperator); + return NULL; + } else { + return pBlock; + } + } + } else { + return pBlock; + } + } +} + +static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SSourceDataInfo dataInfo = {0}; + dataInfo.status = EX_SOURCE_DATA_NOT_READY; + dataInfo.taskId = id; + dataInfo.index = i; + SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + if (pDs == NULL) { + taosArrayDestroy(pInfo->pSourceDataInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { + size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); + + if (numOfSources == 0) { + qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); + return TSDB_CODE_INVALID_PARA; + } + + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + if (pInfo->pSources == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); + taosArrayPush(pInfo->pSources, pNode); + } + + initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); + pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); + + return initDataSource(numOfSources, pInfo, id); +} + +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); + + pInfo->seqLoadData = false; + pInfo->pTransporter = pTransporter; + + pOperator->name = "ExchangeOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + doDestroyExchangeOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + taosRemoveRef(exchangeObjRefPool, pExInfo->self); +} + +void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + +void freeSourceDataInfo(void* p) { + SSourceDataInfo* pInfo = (SSourceDataInfo*)p; + taosMemoryFreeClear(pInfo->pRsp); +} + +void doDestroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + + taosArrayDestroy(pExInfo->pSources); + taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; + } + + blockDataDestroy(pExInfo->pDummyBlock); + + tsem_destroy(&pExInfo->ready); + taosMemoryFreeClear(param); +} + +int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { + SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; + + SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); + if (pExchangeInfo == NULL) { + qWarn("failed to acquire exchange operator, since it may have been released"); + taosMemoryFree(pMsg->pData); + return TSDB_CODE_SUCCESS; + } + + int32_t index = pWrapper->sourceIndex; + SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + + if (code == TSDB_CODE_SUCCESS) { + pSourceDataInfo->pRsp = pMsg->pData; + + SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->compLen = htonl(pRsp->compLen); + pRsp->numOfCols = htonl(pRsp->numOfCols); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); + + ASSERT(pRsp != NULL); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); + } else { + taosMemoryFree(pMsg->pData); + pSourceDataInfo->code = code; + qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); + } + + pSourceDataInfo->status = EX_SOURCE_DATA_READY; + + tsem_post(&pExchangeInfo->ready); + taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); + + return TSDB_CODE_SUCCESS; +} + +int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + + ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); + + SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); + pWrapper->exchangeId = pExchangeInfo->self; + pWrapper->sourceIndex = sourceIndex; + + if (pSource->localExec) { + SDataBuf pBuf = {0}; + int32_t code = + (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + loadRemoteDataCallback(pWrapper, &pBuf, code); + taosMemoryFree(pWrapper); + } else { + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + return pTaskInfo->code; + } + + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, + pSource->execId, sourceIndex, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + pMsg->execId = htonl(pSource->execId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + taosMemoryFreeClear(pMsg); + taosMemoryFree(pWrapper); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } + + pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = pSource->fetchMsgType; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); + } + + return TSDB_CODE_SUCCESS; +} + +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { + if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + *pNextStart = (char*)blockDecode(pRes, pData); + } else { // extract data according to pColList + char* pStart = pData; + + int32_t numOfCols = htonl(*(int32_t*)pStart); + pStart += sizeof(int32_t); + + // todo refactor:extract method + SSysTableSchema* pSchema = (SSysTableSchema*)pStart; + for (int32_t i = 0; i < numOfCols; ++i) { + SSysTableSchema* p = (SSysTableSchema*)pStart; + + p->colId = htons(p->colId); + p->bytes = htonl(p->bytes); + pStart += sizeof(SSysTableSchema); + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); + } + + blockDecode(pBlock, pStart); + blockDataEnsureCapacity(pRes, pBlock->info.rows); + + // data from mnode + pRes->info.rows = pBlock->info.rows; + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); + blockDataDestroy(pBlock); + } + + // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. + blockDataUpdateTsWindow(pRes, 0); + return TSDB_CODE_SUCCESS; +} + +void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + int64_t el = taosGetTimestampUs() - startTs; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + + pLoadInfo->totalElapsed += el; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + + doSetOperatorCompleted(pOperator); + return NULL; +} + +int32_t getCompletedSources(const SArray* pArray) { + size_t total = taosArrayGetSize(pArray); + + int32_t completed = 0; + for (int32_t k = 0; k < total; ++k) { + SSourceDataInfo* p = taosArrayGet(pArray, k); + if (p->status == EX_SOURCE_DATA_EXHAUSTED) { + completed += 1; + } + } + + return completed; +} + +int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for (int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return code; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), + totalSources, (endTs - startTs) / 1000.0); + + pOperator->status = OP_RES_TO_RETURN; + pOperator->cost.openCost = taosGetTimestampUs() - startTs; + + tsem_wait(&pExchangeInfo->ready); + tsem_post(&pExchangeInfo->ready); + return TSDB_CODE_SUCCESS; +} + +int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + while (1) { + if (pExchangeInfo->current >= totalSources) { + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; + } + + doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); + tsem_wait(&pExchangeInfo->ready); + + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), + pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + pOperator->pTaskInfo->code = pDataInfo->code; + return pOperator->pTaskInfo->code; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, + pDataInfo->totalRows, pLoadInfo->totalRows); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + taosMemoryFreeClear(pDataInfo->pRsp); + continue; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); + + if (pRsp->completed == 1) { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, + totalSources); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pLoadInfo->totalRows, pLoadInfo->totalSize); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + + taosMemoryFreeClear(pDataInfo->pRsp); + return TSDB_CODE_SUCCESS; + } +} + +int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + int64_t st = taosGetTimestampUs(); + + SExchangeInfo* pExchangeInfo = pOperator->info; + if (!pExchangeInfo->seqLoadData) { + int32_t code = prepareConcurrentlyLoad(pOperator); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pExchangeInfo->openedTs = taosGetTimestampUs(); + } + + OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + return TSDB_CODE_SUCCESS; +} From 411371ba488dd324f49f19c5ee82a59e094b7bd7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 14:16:21 +0800 Subject: [PATCH 08/56] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +-- source/libs/function/src/builtinsimpl.c | 40 ++++++++++++++++++------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9dab1e1d33..c157faecb1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -973,7 +973,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t mid = dumpedRows >> 1u; int8_t* pts = (int8_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; + int8_t t = pts[j]; pts[j] = pts[dumpedRows - j - 1]; pts[dumpedRows - j - 1] = t; } @@ -998,7 +998,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t mid = dumpedRows >> 1u; int32_t* pts = (int32_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; + int32_t t = pts[j]; pts[j] = pts[dumpedRows - j - 1]; pts[dumpedRows - j - 1] = t; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6eb57c1a18..47977f9757 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3096,18 +3096,36 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } #else int64_t* pts = (int64_t*)pInput->pPTS->pData; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; + + if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + char* data = colDataGetVarData(pInputCol, i); + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } - numOfElems++; + numOfElems++; - char* data = colDataGetData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); - pResInfo->numOfRes = 1; + char* data = colDataGetNumData(pInputCol, i); + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } } } #endif @@ -3266,8 +3284,8 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. - // this is NOT always works if project operator exists in downstream. + // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. + // this assumption is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); From 92dc920f4c8b813962bc991e3e797026f35cfa74 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 14:22:58 +0800 Subject: [PATCH 09/56] perf: change default buffer sizea --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 6e1fa87854..b6a7f5c824 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -290,7 +290,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB #define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB -#define TSDB_DEFAULT_BUFFER_PER_VNODE 96 +#define TSDB_DEFAULT_BUFFER_PER_VNODE 256 #define TSDB_MIN_PAGES_PER_VNODE 64 #define TSDB_MAX_PAGES_PER_VNODE (INT32_MAX - 1) #define TSDB_DEFAULT_PAGES_PER_VNODE 256 From 15e7e345dc40b83084cf5fbdbc7e1a215ca13a5d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 14:45:07 +0800 Subject: [PATCH 10/56] refact: move sync test code to single lib --- source/libs/sync/inc/syncInt.h | 2 - source/libs/sync/inc/syncRaftEntry.h | 34 +- source/libs/sync/inc/syncRaftLog.h | 22 +- source/libs/sync/inc/syncTools.h | 30 +- source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncMain.c | 354 ++--------- source/libs/sync/src/syncMessage.c | 587 ++---------------- source/libs/sync/src/syncRaftEntry.c | 291 +-------- source/libs/sync/src/syncRaftLog.c | 190 +----- source/libs/sync/src/syncReplication.c | 9 +- source/libs/sync/test/CMakeLists.txt | 164 ++--- .../sync/test/syncAppendEntriesBatchTest.cpp | 1 + .../sync/test/syncClientRequestBatchTest.cpp | 125 ---- .../libs/sync/test/syncClientRequestTest.cpp | 98 --- source/libs/sync/test/syncEncodeTest.cpp | 30 +- source/libs/sync/test/syncEntryCacheTest.cpp | 1 + source/libs/sync/test/syncEntryTest.cpp | 18 +- source/libs/sync/test/syncHashCacheTest.cpp | 1 + source/libs/sync/test/syncLogStoreCheck.cpp | 1 + source/libs/sync/test/syncLogStoreCheck2.cpp | 1 + source/libs/sync/test/syncLogStoreTest.cpp | 1 + source/libs/sync/test/syncRaftLogTest2.cpp | 1 + source/libs/sync/test/syncRaftLogTest3.cpp | 9 +- source/libs/sync/test/syncRespMgrTest.cpp | 20 +- source/libs/sync/test/syncRpcMsgTest.cpp | 10 +- source/libs/sync/test/syncSnapshotTest.cpp | 11 +- .../libs/sync/test/syncVotesGrantedTest.cpp | 1 + .../libs/sync/test/syncVotesRespondTest.cpp | 2 + source/libs/sync/test/syncWriteTest.cpp | 18 +- .../sync/test/sync_test_lib/CMakeLists.txt | 14 + .../{ => test/sync_test_lib}/inc/syncIO.h | 2 +- .../sync/test/sync_test_lib/inc/syncTest.h | 90 +++ .../{ => test/sync_test_lib}/src/syncIO.c | 5 +- .../test/sync_test_lib/src/syncMainDebug.c | 255 ++++++++ .../test/sync_test_lib/src/syncMessageDebug.c | 508 +++++++++++++++ .../test/sync_test_lib/src/syncRaftCfgDebug.c | 19 + .../sync_test_lib/src/syncRaftEntryDebug.c | 222 +++++++ .../test/sync_test_lib/src/syncRaftLogDebug.c | 182 ++++++ 38 files changed, 1562 insertions(+), 1769 deletions(-) delete mode 100644 source/libs/sync/test/syncClientRequestBatchTest.cpp delete mode 100644 source/libs/sync/test/syncClientRequestTest.cpp create mode 100644 source/libs/sync/test/sync_test_lib/CMakeLists.txt rename source/libs/sync/{ => test/sync_test_lib}/inc/syncIO.h (95%) create mode 100644 source/libs/sync/test/sync_test_lib/inc/syncTest.h rename source/libs/sync/{ => test/sync_test_lib}/src/syncIO.c (98%) create mode 100644 source/libs/sync/test/sync_test_lib/src/syncMainDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3085b6d8f4..8a951ba38d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -268,8 +268,6 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config); void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index bab1dcc661..8b8ab41b53 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -42,25 +42,14 @@ typedef struct SSyncRaftEntry { char data[]; // origin RpcMsg.pCont } SSyncRaftEntry; -SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); -SSyncRaftEntry* syncEntryBuild4(SRpcMsg* pOriginalMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuild(int32_t dataLen); +SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg); SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId); void syncEntryDestory(SSyncRaftEntry* pEntry); -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 -cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); -char* syncEntry2Str(const SSyncRaftEntry* pEntry); void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 -// for debug ---------------------- -void syncEntryPrint(const SSyncRaftEntry* pObj); -void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); -void syncEntryLog(const SSyncRaftEntry* pObj); -void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); - -//----------------------------------- typedef struct SRaftEntryHashCache { SHashObj* pEntryHash; int32_t maxCount; @@ -78,14 +67,6 @@ int32_t raftCacheDelEntry(struct SRaftEntryHashCache* pCache, SyncI int32_t raftCacheGetAndDel(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); int32_t raftCacheClear(struct SRaftEntryHashCache* pCache); -cJSON* raftCache2Json(SRaftEntryHashCache* pObj); -char* raftCache2Str(SRaftEntryHashCache* pObj); -void raftCachePrint(SRaftEntryHashCache* pObj); -void raftCachePrint2(char* s, SRaftEntryHashCache* pObj); -void raftCacheLog(SRaftEntryHashCache* pObj); -void raftCacheLog2(char* s, SRaftEntryHashCache* pObj); - -//----------------------------------- typedef struct SRaftEntryCache { SSkipList* pSkipList; int32_t maxCount; @@ -102,13 +83,6 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncInde int32_t raftEntryCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count); -cJSON* raftEntryCache2Json(SRaftEntryCache* pObj); -char* raftEntryCache2Str(SRaftEntryCache* pObj); -void raftEntryCachePrint(SRaftEntryCache* pObj); -void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj); -void raftEntryCacheLog(SRaftEntryCache* pObj); -void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index ff59189a9d..c25d4ae34e 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -40,25 +40,19 @@ typedef struct SSyncLogStoreData { SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); void logStoreDestory(SSyncLogStore* pLogStore); -cJSON* logStore2Json(SSyncLogStore* pLogStore); -char* logStore2Str(SSyncLogStore* pLogStore); -cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); -char* logStoreSimple2Str(SSyncLogStore* pLogStore); SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore); -// for debug -void logStorePrint(SSyncLogStore* pLogStore); -void logStorePrint2(char* s, SSyncLogStore* pLogStore); -void logStoreLog(SSyncLogStore* pLogStore); -void logStoreLog2(char* s, SSyncLogStore* pLogStore); - -void logStoreSimplePrint(SSyncLogStore* pLogStore); -void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); -void logStoreSimpleLog(SSyncLogStore* pLogStore); -void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); +SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); +bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); +SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); +SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); +int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); +SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); +SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 193579030f..6a760ecd87 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -180,17 +180,13 @@ typedef struct SyncClientRequest { } SyncClientRequest; SyncClientRequest* syncClientRequestAlloc(uint32_t dataLen); -SyncClientRequest* syncClientRequestBuild(const SRpcMsg* pMsg, uint64_t seqNum, bool isWeak, int32_t vgId); // step 1 -void syncClientRequestDestroy(SyncClientRequest* pMsg); -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 -cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); -char* syncClientRequest2Str(const SyncClientRequest* pMsg); +int32_t syncClientRequestBuildFromRpcMsg(SRpcMsg* pClientRequestRpcMsg, const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, + bool isWeak, int32_t vgId); +int32_t syncClientRequestBuildFromNoopEntry(SRpcMsg* pClientRequestRpcMsg, const SSyncRaftEntry* pEntry, int32_t vgId); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 +void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); +cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); +char* syncClientRequest2Str(const SyncClientRequest* pMsg); // for debug ---------------------- void syncClientRequestPrint(const SyncClientRequest* pMsg); @@ -381,14 +377,6 @@ SyncAppendEntriesBatch* syncAppendEntriesBatchDeserialize2(const char* buf, uint void syncAppendEntriesBatch2RpcMsg(const SyncAppendEntriesBatch* pMsg, SRpcMsg* pRpcMsg); void syncAppendEntriesBatchFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesBatch* pMsg); SyncAppendEntriesBatch* syncAppendEntriesBatchFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg); -char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg); - -// for debug ---------------------- -void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg); // --------------------------------------------- typedef struct SyncAppendEntriesReply { @@ -739,14 +727,14 @@ int32_t syncNodeOnSnapshotReply(SSyncNode* ths, SyncSnapshotRsp* pMsg); int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg); -int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); +int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg); int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg); // ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); -typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7d6e358511..4f3f1c2c00 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -179,7 +179,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->success = true; bool hasAppendEntries = pMsg->dataLen > 0; if (hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + SSyncRaftEntry* pAppendEntry = syncEntryBuildFromAppendEntries(pMsg); ASSERT(pAppendEntry != NULL); SyncIndex appendIndex = pMsg->prevLogIndex + 1; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 14823be098..67d5c5afdd 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -154,9 +154,7 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { code = syncNodeOnPingReply(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest* pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); + code = syncNodeOnClientRequest(pSyncNode, pMsg, NULL); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote* pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); code = syncNodeOnRequestVote(pSyncNode, pSyncMsg); @@ -225,7 +223,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { sError("sync begin snapshot error"); return -1; } - + int32_t code = 0; if (syncNodeIsMnode(pSyncNode)) { @@ -601,44 +599,42 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { return -1; } - int32_t ret = 0; - SyncClientRequest* pSyncMsg; - // optimized one replica if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) { - pSyncMsg = syncClientRequestBuild(pMsg, 0, isWeak, pSyncNode->vgId); - SyncIndex retIndex; - int32_t code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, &retIndex); + int32_t code = syncNodeOnClientRequest(pSyncNode, pMsg, &retIndex); if (code == 0) { pMsg->info.conn.applyIndex = retIndex; pMsg->info.conn.applyTerm = pSyncNode->pRaftStore->currentTerm; - ret = 1; - sTrace("vgId:%d, sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); - } else { - ret = -1; - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, failed to sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, + sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); + return 1; + } else { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("vgId:%d, failed to propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType)); + return -1; } } else { SRespStub stub = {.createTime = taosGetTimestampMs(), .rpcMsg = *pMsg}; uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); - - pSyncMsg = syncClientRequestBuild(pMsg, seqNum, isWeak, pSyncNode->vgId); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - - sNTrace(pSyncNode, "propose message, type:%s", TMSG_INFO(pMsg->msgType)); - ret = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); - if (ret != 0) { - sError("vgId:%d, failed to enqueue msg since %s", pSyncNode->vgId, terrstr()); - syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + SRpcMsg rpcMsg = {0}; + int32_t code = syncClientRequestBuildFromRpcMsg(&rpcMsg, pMsg, seqNum, isWeak, pSyncNode->vgId); + if (code != 0) { + sError("vgId:%d, failed to propose msg while serialize since %s", pSyncNode->vgId, terrstr()); + (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + return -1; } - } - syncClientRequestDestroy(pSyncMsg); - return ret; + sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType)); + code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); + (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + } + + return code; + } } static int32_t syncHbTimerInit(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer, SRaftId destId) { @@ -1084,53 +1080,6 @@ void syncNodeClose(SSyncNode* pSyncNode) { ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; } -// ping -------------- -int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { - syncPingLog2((char*)"==syncNodePing==", pMsg); - int32_t ret = 0; - - SRpcMsg rpcMsg; - syncPing2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); - - ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); - return ret; -} - -int32_t syncNodePingSelf(SSyncNode* pSyncNode) { - int32_t ret = 0; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); - ASSERT(ret == 0); - - syncPingDestroy(pMsg); - return ret; -} - -int32_t syncNodePingPeers(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId* destId = &(pSyncNode->peersId[i]); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, destId, pMsg); - ASSERT(ret == 0); - syncPingDestroy(pMsg); - } - return ret; -} - -int32_t syncNodePingAll(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - SRaftId* destId = &(pSyncNode->replicasId[i]); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, destId, pMsg); - ASSERT(ret == 0); - syncPingDestroy(pMsg); - } - return ret; -} - // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; @@ -1293,196 +1242,6 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S return 0; } -cJSON* syncNode2Json(const SSyncNode* pSyncNode) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pSyncNode != NULL) { - // init by SSyncInfo - cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); - cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); - cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); - cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); - cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); - cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncSendMSg); - cJSON_AddStringToObject(pRoot, "syncSendMSg", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); - cJSON_AddStringToObject(pRoot, "queue", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncEqMsg); - cJSON_AddStringToObject(pRoot, "syncEqMsg", u64buf); - - // init internal - cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); - cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); - cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); - cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); - - cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); - cJSON* pPeers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); - } - cJSON* pPeersId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersId", pPeersId); - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); - } - - cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); - cJSON* pReplicasId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); - } - - // raft algorithm - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); - cJSON_AddStringToObject(pRoot, "pFsm", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); - cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); - cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); - - // life cycle - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); - cJSON_AddStringToObject(pRoot, "rid", u64buf); - - // tla+ server vars - cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); - cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); - cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); - - // tla+ candidate vars - cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); - cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); - - // tla+ leader vars - cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); - cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); - - // tla+ log vars - cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - // timer ms init - cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); - cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); - cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); - - // ping timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); - cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); - cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); - cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); - - // elect timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); - cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); - cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); - cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); - cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); - - // heartbeat timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); - cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); - cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); - cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); - - // callback - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); - cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); - cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); - cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); - cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); - cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); - - // restoreFinish - cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); - - // snapshot senders - cJSON* pSenders = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "senders", pSenders); - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); - } - - // snapshot receivers - cJSON* pReceivers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); - - // changing - cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); - return pJson; -} - -char* syncNode2Str(const SSyncNode* pSyncNode) { - cJSON* pJson = syncNode2Json(pSyncNode); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { - int32_t len = 256; - char* s = (char*)taosMemoryMalloc(len); - - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); - - snprintf(s, len, - "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 - ", sby:%d, " - "r-num:%d, " - "lcfg:%" PRId64 ", chging:%d, rsto:%d", - pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, - logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, - pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); - - return s; -} - inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { bool b1 = false; bool b2 = false; @@ -2056,8 +1815,6 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex return 0; } -// ------ local funciton --------- -// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { @@ -2065,7 +1822,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { pSyncNode->pingTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); + sNTrace(pSyncNode, "enqueue ping timer"); if (pSyncNode->syncEqMsg != NULL) { int32_t code = pSyncNode->syncEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { @@ -2141,7 +1898,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); + sNTrace(pSyncNode, "enqueue heartbeat timer"); if (pSyncNode->syncEqMsg != NULL) { int32_t code = pSyncNode->syncEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { @@ -2230,34 +1987,28 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { } } -static int32_t syncNodeEqNoop(SSyncNode* ths) { - int32_t ret = 0; - ASSERT(ths->state == TAOS_SYNC_STATE_LEADER); - - SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); - ASSERT(pEntry != NULL); - - uint32_t entryLen; - char* serialized = syncEntrySerialize(pEntry, &entryLen); - SyncClientRequest* pSyncMsg = syncClientRequestAlloc(entryLen); - ASSERT(pSyncMsg->dataLen == entryLen); - memcpy(pSyncMsg->data, serialized, entryLen); - - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - if (ths->syncEqMsg != NULL) { - ths->syncEqMsg(ths->msgcb, &rpcMsg); - } else { - sTrace("syncNodeEqNoop pSyncNode->syncEqMsg is NULL"); +static int32_t syncNodeEqNoop(SSyncNode* pNode) { + if (pNode->state == TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + return -1; } - syncEntryDestory(pEntry); - taosMemoryFree(serialized); - syncClientRequestDestroy(pSyncMsg); + SyncIndex index = pNode->pLogStore->syncLogWriteIndex(pNode->pLogStore); + SyncTerm term = pNode->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, pNode->vgId); + if (pEntry == NULL) return -1; - return ret; + SRpcMsg rpcMsg = {0}; + int32_t code = syncClientRequestBuildFromNoopEntry(&rpcMsg, pEntry, pNode->vgId); + syncEntryDestory(pEntry); + + sNTrace(pNode, "propose msg, type:noop"); + code = (*pNode->syncEqMsg)(pNode->msgcb, &rpcMsg); + if (code != 0) { + sNError(pNode, "failed to propose noop msg while enqueue since %s", terrstr()); + } + + return code; } static void deleteCacheEntry(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } @@ -2438,7 +2189,7 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg) { // leaderVars, commitIndex>> // -int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { +int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) { sNTrace(ths, "on client request"); int32_t ret = 0; @@ -2446,8 +2197,13 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuild2(pMsg, term, index); - ASSERT(pEntry != NULL); + SSyncRaftEntry* pEntry; + + if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index); + } else { + pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index); + } LRUHandle* h = NULL; syncCacheEntry(ths->pLogStore, pEntry, &h); @@ -2646,7 +2402,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde } } - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); // user commit diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 6453693576..3fcb563f3b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -19,146 +19,6 @@ #include "syncUtil.h" #include "tcoding.h" -// --------------------------------------------- -cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { - cJSON* pRoot; - - // in compiler optimization, switch case = if else constants - if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncTimeout2Json(pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { - SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncPing2Json(pSyncMsg); - syncPingDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncPingReply2Json(pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest* pSyncMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncClientRequest2Json(pSyncMsg); - syncClientRequestDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { - pRoot = syncRpcUnknownMsg2Json(); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncRequestVote2Json(pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncRequestVoteReply2Json(pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncAppendEntries2Json(pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncAppendEntriesReply2Json(pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { - SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncSnapshotSend2Json(pSyncMsg); - syncSnapshotSendDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { - SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncSnapshotRsp2Json(pSyncMsg); - syncSnapshotRspDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { - SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncLeaderTransfer2Json(pSyncMsg); - syncLeaderTransferDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { - pRoot = cJSON_CreateObject(); - char* s; - s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont2", s); - taosMemoryFree(s); - - } else { - pRoot = cJSON_CreateObject(); - char* s; - s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont2", s); - taosMemoryFree(s); - } - - cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); - cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); - cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); - // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); - return pJson; -} - -cJSON* syncRpcUnknownMsg2Json() { - cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); - cJSON_AddStringToObject(pRoot, "data", "unknown message"); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); - return pJson; -} - -char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { - cJSON* pJson = syncRpcMsg2Json(pRpcMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncRpcMsgPrint(SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncRpcMsgLog(SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); @@ -840,69 +700,49 @@ SyncClientRequest* syncClientRequestAlloc(uint32_t dataLen) { return pMsg; } -// step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak -SyncClientRequest* syncClientRequestBuild(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak, int32_t vgId) { - SyncClientRequest* pMsg = syncClientRequestAlloc(pOriginalRpcMsg->contLen); - pMsg->vgId = vgId; - pMsg->originalRpcType = pOriginalRpcMsg->msgType; - pMsg->seqNum = seqNum; - pMsg->isWeak = isWeak; - memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); - return pMsg; -} - -void syncClientRequestDestroy(SyncClientRequest* pMsg) { - if (pMsg != NULL) { - taosMemoryFree(pMsg); +int32_t syncClientRequestBuildFromRpcMsg(SRpcMsg* pClientRequestRpcMsg, const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, + bool isWeak, int32_t vgId) { + int32_t bytes = sizeof(SyncClientRequest) + pOriginalRpcMsg->contLen; + pClientRequestRpcMsg->pCont = rpcMallocCont(bytes); + if (pClientRequestRpcMsg->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + + SyncClientRequest* pClientRequest = pClientRequestRpcMsg->pCont; + pClientRequest->bytes = bytes; + pClientRequest->vgId = vgId; + pClientRequest->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequest->originalRpcType = pOriginalRpcMsg->msgType; + pClientRequest->seqNum = seqNum; + pClientRequest->isWeak = isWeak; + pClientRequest->dataLen = pOriginalRpcMsg->contLen; + memcpy(pClientRequest->data, (char*)pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); + + pClientRequestRpcMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequestRpcMsg->contLen = bytes; + return 0; } -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen) { - ASSERT(pMsg->bytes <= bufLen); - memcpy(buf, pMsg, pMsg->bytes); -} - -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg) { - memcpy(pMsg, buf, len); - ASSERT(len == pMsg->bytes); -} - -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { - char* buf = taosMemoryMalloc(pMsg->bytes); - ASSERT(buf != NULL); - syncClientRequestSerialize(pMsg, buf, pMsg->bytes); - if (len != NULL) { - *len = pMsg->bytes; +int32_t syncClientRequestBuildFromNoopEntry(SRpcMsg* pClientRequestRpcMsg, const SSyncRaftEntry* pEntry, int32_t vgId) { + int32_t bytes = sizeof(SyncClientRequest) + pEntry->bytes; + pClientRequestRpcMsg->pCont = rpcMallocCont(bytes); + if (pClientRequestRpcMsg->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - return buf; -} -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { - uint32_t bytes = *((uint32_t*)buf); - SyncClientRequest* pMsg = taosMemoryMalloc(bytes); - ASSERT(pMsg != NULL); - syncClientRequestDeserialize(buf, len, pMsg); - ASSERT(len == pMsg->bytes); - return pMsg; -} + SyncClientRequest* pClientRequest = pClientRequestRpcMsg->pCont; + pClientRequest->bytes = bytes; + pClientRequest->vgId = vgId; + pClientRequest->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequest->originalRpcType = TDMT_SYNC_NOOP; + pClientRequest->dataLen = pEntry->bytes; + memcpy(pClientRequest->data, (char*)pEntry, pEntry->bytes); -// step 2. SyncClientRequest => RpcMsg, to queue -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) { - pRpcMsg->msgType = pMsg->msgType; - pRpcMsg->contLen = pMsg->bytes; - pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - syncClientRequestSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); -} - -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg) { - syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); -} - -// step 3. RpcMsg => SyncClientRequest, from queue -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg) { - SyncClientRequest* pMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - ASSERT(pMsg != NULL); - return pMsg; + pClientRequestRpcMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequestRpcMsg->contLen = bytes; + return 0; } cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { @@ -940,35 +780,6 @@ char* syncClientRequest2Str(const SyncClientRequest* pMsg) { return serialized; } -// for debug ---------------------- -void syncClientRequestPrint(const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestLog(const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncClientRequestBatch---- // block1: @@ -1059,86 +870,6 @@ SyncClientRequestBatch* syncClientRequestBatchFromRpcMsg(const SRpcMsg* pRpcMsg) return pSyncMsg; } -cJSON* syncClientRequestBatch2Json(const SyncClientRequestBatch* pMsg) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); - - SRaftMeta* metaArr = syncClientRequestBatchMetaArr(pMsg); - SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pMsg); - - cJSON* pMetaArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pMeta = cJSON_CreateObject(); - cJSON_AddNumberToObject(pMeta, "seqNum", metaArr[i].seqNum); - cJSON_AddNumberToObject(pMeta, "isWeak", metaArr[i].isWeak); - cJSON_AddItemToArray(pMetaArr, pMeta); - } - - cJSON* pMsgArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "msgArr", pMsgArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pRpcMsgJson = syncRpcMsg2Json(&msgArr[i]); - cJSON_AddItemToArray(pMsgArr, pRpcMsgJson); - } - - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncClientRequestBatch", pRoot); - return pJson; -} - -char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { - cJSON* pJson = syncClientRequestBatch2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - sTrace("syncClientRequestBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncClientRequestBatch2Str(pMsg); - sLTrace("syncClientRequestBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncRequestVote); @@ -1716,138 +1447,6 @@ SyncAppendEntriesBatch* syncAppendEntriesBatchFromRpcMsg2(const SRpcMsg* pRpcMsg return pMsg; } -cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); - cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - - int32_t metaArrayLen = sizeof(SOffsetAndContLen) * pMsg->dataCount; // - int32_t entryArrayLen = pMsg->dataLen - metaArrayLen; - - cJSON_AddNumberToObject(pRoot, "metaArrayLen", metaArrayLen); - cJSON_AddNumberToObject(pRoot, "entryArrayLen", entryArrayLen); - - SOffsetAndContLen* metaArr = (SOffsetAndContLen*)(pMsg->data); - - cJSON* pMetaArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pMeta = cJSON_CreateObject(); - cJSON_AddNumberToObject(pMeta, "offset", metaArr[i].offset); - cJSON_AddNumberToObject(pMeta, "contLen", metaArr[i].contLen); - cJSON_AddItemToArray(pMetaArr, pMeta); - } - - cJSON* pEntryArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entryArr", pEntryArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)(pMsg->data + metaArr[i].offset); - cJSON* pEntryJson = syncEntry2Json(pEntry); - cJSON_AddItemToArray(pEntryArr, pEntryJson); - } - - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncAppendEntriesBatch", pRoot); - return pJson; -} - -char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { - cJSON* pJson = syncAppendEntriesBatch2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - sTrace("syncAppendEntriesBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - sLTrace("syncAppendEntriesBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncAppendEntriesReply---- SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncAppendEntriesReply); @@ -2827,118 +2426,6 @@ SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) { return pMsg; } -cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { - char u64buf[128]; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); - cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); - cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); - cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); - cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); - - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); - return pJson; -} - -char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { - cJSON* pJson = syncSnapshotSend2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - -// --------------------------------------------- SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncSnapshotRsp); SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 520ecd95db..c5b1399c8c 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,31 +13,28 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncRaftEntry.h" #include "syncUtil.h" -SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { - uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; +SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { + int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - ASSERT(pEntry != NULL); - memset(pEntry, 0, bytes); + if (pEntry == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pEntry->bytes = bytes; pEntry->dataLen = dataLen; pEntry->rid = -1; - return pEntry; -} - -// step 4. SyncClientRequest => SSyncRaftEntry, add term, index -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index); - ASSERT(pEntry != NULL); return pEntry; } -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { +SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); - ASSERT(pEntry != NULL); + if (pEntry == NULL) return NULL; pEntry->msgType = pMsg->msgType; pEntry->originalRpcType = pMsg->originalRpcType; @@ -45,42 +42,37 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; - pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); return pEntry; } -SSyncRaftEntry* syncEntryBuild4(SRpcMsg* pOriginalMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild(pOriginalMsg->contLen); - ASSERT(pEntry != NULL); +SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->contLen); + if (pEntry == NULL) return NULL; pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; - pEntry->originalRpcType = pOriginalMsg->msgType; + pEntry->originalRpcType = pMsg->msgType; pEntry->seqNum = 0; pEntry->isWeak = 0; pEntry->term = term; pEntry->index = index; - pEntry->dataLen = pOriginalMsg->contLen; - memcpy(pEntry->data, pOriginalMsg->pCont, pOriginalMsg->contLen); + memcpy(pEntry->data, pMsg->pCont, pMsg->contLen); return pEntry; } -SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { - // init rpcMsg - SMsgHead head; - head.vgId = vgId; - head.contLen = sizeof(SMsgHead); - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(SRpcMsg)); - rpcMsg.contLen = head.contLen; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - rpcMsg.msgType = TDMT_SYNC_NOOP; - memcpy(rpcMsg.pCont, &head, sizeof(head)); +SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); + if (pEntry == NULL) return NULL; - SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen); - ASSERT(pEntry != NULL); + memcpy(pEntry, pMsg->data, pMsg->dataLen); + return pEntry; +} + +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { + SSyncRaftEntry* pEntry = syncEntryBuild(sizeof(SMsgHead)); + if (pEntry == NULL) return NULL; pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; pEntry->originalRpcType = TDMT_SYNC_NOOP; @@ -89,9 +81,9 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) pEntry->term = term; pEntry->index = index; - ASSERT(pEntry->dataLen == rpcMsg.contLen); - memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen); - rpcFreeCont(rpcMsg.pCont); + SMsgHead* pHead = (SMsgHead*)pEntry->data; + pHead->vgId = vgId; + pHead->contLen = sizeof(SMsgHead); return pEntry; } @@ -102,104 +94,13 @@ void syncEntryDestory(SSyncRaftEntry* pEntry) { } } -// step 5. SSyncRaftEntry => bin, to raft log -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { - char* buf = taosMemoryMalloc(pEntry->bytes); - ASSERT(buf != NULL); - memcpy(buf, pEntry, pEntry->bytes); - if (len != NULL) { - *len = pEntry->bytes; - } - return buf; -} - -// step 6. bin => SSyncRaftEntry, from raft log -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { - uint32_t bytes = *((uint32_t*)buf); - SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - ASSERT(pEntry != NULL); - memcpy(pEntry, buf, len); - ASSERT(len == pEntry->bytes); - return pEntry; -} - -cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pEntry != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); - cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); - cJSON_AddStringToObject(pRoot, "seqNum", u64buf); - cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); - cJSON_AddStringToObject(pRoot, "index", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->rid); - cJSON_AddStringToObject(pRoot, "rid", u64buf); - cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); - - char* s; - s = syncUtilPrintBin((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - - s = syncUtilPrintBin2((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); - return pJson; -} - -char* syncEntry2Str(const SSyncRaftEntry* pEntry) { - cJSON* pJson = syncEntry2Json(pEntry); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// step 7. SSyncRaftEntry => original SRpcMsg, commit to user, delete seqNum, isWeak, term, index void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { - memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pEntry->originalRpcType; pRpcMsg->contLen = pEntry->dataLen; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); } -// for debug ---------------------- -void syncEntryPrint(const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncEntryLog(const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); -} - //----------------------------------- SRaftEntryHashCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { SRaftEntryHashCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryHashCache)); @@ -354,76 +255,6 @@ int32_t raftCacheClear(struct SRaftEntryHashCache* pCache) { return 0; } -//----------------------------------- -cJSON* raftCache2Json(SRaftEntryHashCache* pCache) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - - snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); - cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entries", pEntries); - - SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); - if (pIter != NULL) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - while (pIter) { - pIter = taosHashIterate(pCache->pEntryHash, pIter); - if (pIter != NULL) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - } - - taosThreadMutexUnlock(&pCache->mutex); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftEntryHashCache", pRoot); - return pJson; -} - -char* raftCache2Str(SRaftEntryHashCache* pCache) { - cJSON* pJson = raftCache2Json(pCache); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -void raftCachePrint(SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - printf("raftCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftCachePrint2(char* s, SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - printf("raftCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftCacheLog(SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - sTrace("raftCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void raftCacheLog2(char* s, SRaftEntryHashCache* pCache) { - if (gRaftDetailLog) { - char* serialized = raftCache2Str(pCache); - sLTrace("raftCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} //----------------------------------- static char* keyFn(const void* pData) { @@ -612,69 +443,3 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) { taosThreadMutexUnlock(&pCache->mutex); return returnCnt; } - -cJSON* raftEntryCache2Json(SRaftEntryCache* pCache) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - - snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); - cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entries", pEntries); - - SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); - while (tSkipListIterNext(pIter)) { - SSkipListNode* pNode = tSkipListIterGet(pIter); - ASSERT(pNode != NULL); - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - tSkipListDestroyIter(pIter); - - taosThreadMutexUnlock(&pCache->mutex); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); - return pJson; -} - -char* raftEntryCache2Str(SRaftEntryCache* pObj) { - cJSON* pJson = raftEntryCache2Json(pObj); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -void raftEntryCachePrint(SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - printf("raftEntryCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - printf("raftEntryCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftEntryCacheLog(SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - sTrace("raftEntryCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj) { - if (gRaftDetailLog) { - char* serialized = raftEntryCache2Str(pObj); - sLTrace("raftEntryCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d3d69b288b..85c94884f2 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -22,15 +22,8 @@ // public function static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex); -static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); -static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); -static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); -static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); + static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); -static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); static bool raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index); static int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); @@ -126,29 +119,29 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI return 0; } -static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex firstVer = walGetFirstVer(pWal); return firstVer; } -static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } +SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } -static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { +bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; return walIsEmpty(pWal); } -static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { +int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore); int32_t count = endIndex - beginIndex + 1; return count > 0 ? count : 0; } -static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { SyncIndex lastIndex; SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -157,7 +150,7 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { return lastVer; } -static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex lastVer = walGetLastVer(pWal); @@ -174,7 +167,7 @@ static bool raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index) { // if success, return last term // if not log, return 0 // if error, return SYNC_TERM_INVALID -static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { +SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (walIsEmpty(pWal)) { @@ -225,7 +218,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr // entry found, return 0 // entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST // other error, return -1 -static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; int32_t code = 0; @@ -364,111 +357,6 @@ SyncIndex raftlogCommitIndex(SSyncLogStore* pLogStore) { return pData->pSyncNode->commitIndex; } -cJSON* logStore2Json(SSyncLogStore* pLogStore) { - char u64buf[128] = {0}; - SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; - cJSON* pRoot = cJSON_CreateObject(); - - if (pData != NULL && pData->pWal != NULL) { - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); - cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - - int32_t count = raftLogEntryCount(pLogStore); - cJSON_AddNumberToObject(pRoot, "entryCount", count); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); - cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "pEntries", pEntries); - - if (!raftLogIsEmpty(pLogStore)) { - for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - SSyncRaftEntry* pEntry = NULL; - raftLogGetEntry(pLogStore, i, &pEntry); - - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - syncEntryDestory(pEntry); - } - } - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); - return pJson; -} - -char* logStore2Str(SSyncLogStore* pLogStore) { - cJSON* pJson = logStore2Json(pLogStore); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { - char u64buf[128] = {0}; - SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; - cJSON* pRoot = cJSON_CreateObject(); - - if (pData != NULL && pData->pWal != NULL) { - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); - cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - - int32_t count = raftLogEntryCount(pLogStore); - cJSON_AddNumberToObject(pRoot, "entryCount", count); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); - cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); - return pJson; -} - -char* logStoreSimple2Str(SSyncLogStore* pLogStore) { - cJSON* pJson = logStoreSimple2Json(pLogStore); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -480,63 +368,3 @@ SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore) { SWal* pWal = pData->pWal; return walGetCommittedVer(pWal); } - -// for debug ----------------- -void logStorePrint(SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - printf("logStorePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStorePrint2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - printf("logStorePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreLog(SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStore2Str(pLogStore); - sLTrace("logStoreLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); - } -} - -void logStoreLog2(char* s, SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStore2Str(pLogStore); - sLTrace("logStoreLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - -// for debug ----------------- -void logStoreSimplePrint(SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreSimpleLog(SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 7b4ef63406..8c3f7e5384 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -78,14 +78,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) { pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); ASSERT(pMsg != NULL); - - // add pEntry into msg - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - ASSERT(len == pEntry->bytes); - memcpy(pMsg->data, serialized, len); - - taosMemoryFree(serialized); + memcpy(pMsg->data, pEntry->data, pEntry->bytes); syncEntryDestory(pEntry); } else { diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index e584893ae0..7a22d96972 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(sync_test_lib) add_executable(syncTest "") add_executable(syncRaftIdCheck "") add_executable(syncEnvTest "") @@ -24,8 +25,6 @@ add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") add_executable(syncAppendEntriesBatchTest "") add_executable(syncAppendEntriesReplyTest "") -add_executable(syncClientRequestTest "") -add_executable(syncClientRequestBatchTest "") add_executable(syncTimeoutTest "") add_executable(syncPingTest "") add_executable(syncPingReplyTest "") @@ -167,14 +166,6 @@ target_sources(syncAppendEntriesReplyTest PRIVATE "syncAppendEntriesReplyTest.cpp" ) -target_sources(syncClientRequestTest - PRIVATE - "syncClientRequestTest.cpp" -) -target_sources(syncClientRequestBatchTest - PRIVATE - "syncClientRequestBatchTest.cpp" -) target_sources(syncTimeoutTest PRIVATE "syncTimeoutTest.cpp" @@ -287,10 +278,6 @@ target_sources(syncLeaderTransferTest PRIVATE "syncLeaderTransferTest.cpp" ) -target_sources(syncReconfigFinishTest - PRIVATE - "syncReconfigFinishTest.cpp" -) target_sources(syncRestoreFromSnapshot PRIVATE "syncRestoreFromSnapshot.cpp" @@ -451,16 +438,6 @@ target_include_directories(syncAppendEntriesReplyTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncClientRequestTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncClientRequestBatchTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncTimeoutTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -601,11 +578,6 @@ target_include_directories(syncLeaderTransferTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncReconfigFinishTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncRestoreFromSnapshot PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -644,259 +616,247 @@ target_include_directories(syncPreSnapshotReplyTest target_link_libraries(syncTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftIdCheck - sync + sync_test_lib gtest_main ) target_link_libraries(syncEnvTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTimerTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOTickQTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOTickPingTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOSendMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOClientTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOServerTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftStoreTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEnqTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIndexTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncInitTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncUtilTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncVotesGrantedTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncVotesRespondTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIndexMgrTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEntryTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEntryCacheTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHashCacheTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRequestVoteTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRequestVoteReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesBatchTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesReplyTest - sync - gtest_main -) -target_link_libraries(syncClientRequestTest - sync - gtest_main -) -target_link_libraries(syncClientRequestBatchTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncTimeoutTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRpcMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTimerTest2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingSelfTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncElectTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEncodeTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncWriteTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncReplicateTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRefTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreCheck - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreCheck2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftCfgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRespMgrTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncApplyMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncConfigChangeTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncConfigChangeSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotSendTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotRspTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotSenderTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotReceiverTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncTestTool - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest3 - sync + sync_test_lib gtest_main ) target_link_libraries(syncLeaderTransferTest - sync - gtest_main -) -target_link_libraries(syncReconfigFinishTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRestoreFromSnapshot - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftCfgIndexTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHeartbeatTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHeartbeatReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLocalCmdTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPreSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPreSnapshotReplyTest - sync + sync_test_lib gtest_main ) diff --git a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp index f2544d8fec..98b8654734 100644 --- a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftEntry.h" #include "syncUtil.h" #include "trpc.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncClientRequestBatchTest.cpp b/source/libs/sync/test/syncClientRequestBatchTest.cpp deleted file mode 100644 index f07ee08b2b..0000000000 --- a/source/libs/sync/test/syncClientRequestBatchTest.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -SRpcMsg *createRpcMsg(int32_t i, int32_t dataLen) { - SyncPing *pSyncMsg = syncPingBuild(20); - snprintf(pSyncMsg->data, pSyncMsg->dataLen, "value_%d", i); - - SRpcMsg *pRpcMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); - memset(pRpcMsg, 0, sizeof(SRpcMsg)); - pRpcMsg->code = 10 * i; - syncPing2RpcMsg(pSyncMsg, pRpcMsg); - - syncPingDestroy(pSyncMsg); - return pRpcMsg; -} - -SyncClientRequestBatch *createMsg() { - SRpcMsg *rpcMsgPArr[5]; - memset(rpcMsgPArr, 0, sizeof(rpcMsgPArr)); - for (int32_t i = 0; i < 5; ++i) { - SRpcMsg *pRpcMsg = createRpcMsg(i, 20); - rpcMsgPArr[i] = pRpcMsg; - // taosMemoryFree(pRpcMsg); - } - - SRaftMeta raftArr[5]; - memset(raftArr, 0, sizeof(raftArr)); - for (int32_t i = 0; i < 5; ++i) { - raftArr[i].seqNum = i * 10; - raftArr[i].isWeak = i % 2; - } - - SyncClientRequestBatch *pMsg = syncClientRequestBatchBuild(rpcMsgPArr, raftArr, 5, 1234); - return pMsg; -} - -void test1() { - SyncClientRequestBatch *pMsg = createMsg(); - syncClientRequestBatchLog2((char *)"==test1==", pMsg); - syncClientRequestBatchDestroyDeep(pMsg); -} - -/* -void test2() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); - syncClientRequestSerialize(pMsg, serialized, len); - SyncClientRequest *pMsg2 = syncClientRequestAlloc(pMsg->dataLen); - syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test3() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len; - char * serialized = syncClientRequestSerialize2(pMsg, &len); - SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test4() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); - syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test5() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} -*/ - -int main() { - gRaftDetailLog = true; - tsAsyncLog = 0; - sDebugFlag = DEBUG_DEBUG + DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - logTest(); - - test1(); - - /* -test2(); -test3(); -test4(); -test5(); -*/ - - return 0; -} diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp deleted file mode 100644 index b6bfcc2da5..0000000000 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -SyncClientRequest *createMsg() { - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.msgType = 12345; - rpcMsg.contLen = 20; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild(&rpcMsg, 123, true, 1000); - rpcFreeCont(rpcMsg.pCont); - return pMsg; -} - -void test1() { - SyncClientRequest *pMsg = createMsg(); - syncClientRequestLog2((char *)"test1:", pMsg); - syncClientRequestDestroy(pMsg); -} - -void test2() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char *serialized = (char *)taosMemoryMalloc(len); - syncClientRequestSerialize(pMsg, serialized, len); - SyncClientRequest *pMsg2 = syncClientRequestAlloc(pMsg->dataLen); - syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test3() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len; - char *serialized = syncClientRequestSerialize2(pMsg, &len); - SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test4() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); - syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test5() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -int main() { - tsAsyncLog = 0; - sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - logTest(); - - test1(); - test2(); - test3(); - test4(); - test5(); - - return 0; -} diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index c60176fbf8..216291c1d8 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -7,9 +7,11 @@ #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" +#if 0 void logTest() { sTrace("--- sync log test: trace"); sDebug("--- sync log test: debug"); @@ -118,17 +120,7 @@ SyncClientRequest *step3(const SRpcMsg *pMsg) { } SSyncRaftEntry *step4(const SyncClientRequest *pMsg) { - SSyncRaftEntry *pRetMsg = syncEntryBuild2((SyncClientRequest *)pMsg, 100, 0); - return pRetMsg; -} - -char *step5(const SSyncRaftEntry *pMsg, uint32_t *len) { - char *pRetMsg = syncEntrySerialize(pMsg, len); - return pRetMsg; -} - -SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { - SSyncRaftEntry *pRetMsg = syncEntryDeserialize(pMsg, len); + SSyncRaftEntry *pRetMsg = syncEntryBuildFromClientRequest((SyncClientRequest *)pMsg, 100, 0); return pRetMsg; } @@ -137,13 +129,14 @@ SRpcMsg *step7(const SSyncRaftEntry *pMsg) { syncEntry2OriginalRpc(pMsg, pRetMsg); return pRetMsg; } - +#endif int main(int argc, char **argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; void logTest(); +#if 0 myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); @@ -188,20 +181,9 @@ int main(int argc, char **argv) { syncEntryLog2((char *)"==pEntry==", pEntry); - // step5 - uint32_t len; - char *pMsg5 = step5(pMsg4, &len); - char *s = syncUtilPrintBin(pMsg5, len); - printf("==step5== [%s] \n", s); - taosMemoryFree(s); - - // step6 - SSyncRaftEntry *pMsg6 = step6(pMsg5, len); - syncEntryLog2((char *)"==step6==", pMsg6); - // step7 SRpcMsg *pMsg7 = step7(pMsg6); syncRpcMsgLog2((char *)"==step7==", pMsg7); - +#endif return 0; } diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp index 3ee28ce96b..56b3d6da2e 100644 --- a/source/libs/sync/test/syncEntryCacheTest.cpp +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -7,6 +7,7 @@ #include "syncUtil.h" #include "tref.h" #include "tskiplist.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index b274408c01..e94755195b 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -38,10 +39,10 @@ void test2() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test2"); - SSyncRaftEntry* pEntry = syncEntryBuild2(pSyncMsg, 100, 200); + SSyncRaftEntry* pEntry = syncEntryBuildFromClientRequest(pSyncMsg, 100, 200); syncEntryPrint(pEntry); - syncClientRequestDestroy(pSyncMsg); + taosMemoryFree(pSyncMsg); syncEntryDestory(pEntry); } @@ -52,10 +53,10 @@ void test3() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test3"); - SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200); + SSyncRaftEntry* pEntry = syncEntryBuildFromClientRequest(pSyncMsg, 100, 200); syncEntryPrint(pEntry); - syncClientRequestDestroy(pSyncMsg); + taosMemoryFree(pSyncMsg); syncEntryDestory(pEntry); } @@ -71,14 +72,7 @@ void test4() { strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - assert(serialized != NULL); - SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); - syncEntryPrint(pEntry2); - - taosMemoryFree(serialized); - syncEntryDestory(pEntry2); + // syncEntryDestory(pEntry2); syncEntryDestory(pEntry); } diff --git a/source/libs/sync/test/syncHashCacheTest.cpp b/source/libs/sync/test/syncHashCacheTest.cpp index 7d822971da..2f5bb07f9a 100644 --- a/source/libs/sync/test/syncHashCacheTest.cpp +++ b/source/libs/sync/test/syncHashCacheTest.cpp @@ -4,6 +4,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "tskiplist.h" diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 431b291ca7..0161160a75 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp index 80679bc85c..29ad0610e7 100644 --- a/source/libs/sync/test/syncLogStoreCheck2.cpp +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -7,6 +7,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 9ff0ed2089..832b42bf80 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 78c08a6d8d..3d50b63ff9 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index cf862feb2a..31c06625aa 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRespMgrTest.cpp b/source/libs/sync/test/syncRespMgrTest.cpp index cad6eec91d..8d709e8c81 100644 --- a/source/libs/sync/test/syncRespMgrTest.cpp +++ b/source/libs/sync/test/syncRespMgrTest.cpp @@ -1,8 +1,4 @@ -#include "syncRespMgr.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -64,13 +60,13 @@ void syncRespMgrGetTest(uint64_t i) { void syncRespMgrGetAndDelTest(uint64_t i) { printf("------syncRespMgrGetAndDelTest-------%" PRIu64 "-- \n", i); - SRespStub stub; - int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); - if (ret == 1) { - printStub(&stub); - } else if (ret == 0) { - printf("%" PRId64 " notFound \n", i); - } + // SRespStub stub; + // int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); + // if (ret == 1) { + // printStub(&stub); + // } else if (ret == 0) { + // printf("%" PRId64 " notFound \n", i); + // } } SSyncNode *createSyncNode() { diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 941fa7eab5..127d8e1c41 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -47,7 +47,11 @@ SyncClientRequest *createSyncClientRequest() { rpcMsg.contLen = 20; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild(&rpcMsg, 123, true, 1000); + + SRpcMsg clientRequestMsg; + syncClientRequestBuildFromRpcMsg(&clientRequestMsg, &rpcMsg, 123, true, 1000); + SyncClientRequest *pMsg = (SyncClientRequest *)taosMemoryMalloc(clientRequestMsg.contLen); + memcpy(pMsg->data, clientRequestMsg.pCont, clientRequestMsg.contLen); return pMsg; } @@ -155,11 +159,13 @@ void test7() { } void test8() { +#if 0 SyncClientRequest *pMsg = createSyncClientRequest(); SRpcMsg rpcMsg = {0}; syncClientRequest2RpcMsg(pMsg, &rpcMsg); syncRpcMsgLog2((char *)"test8", &rpcMsg); - syncClientRequestDestroy(pMsg); + taosMemoryFree(pMsg); +#endif } int main() { diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index a1cedd624a..e2264fd08c 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -162,8 +162,11 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild(pMsg, 123, true, 1000); - return pRetMsg; + SRpcMsg clientRequestMsg; + syncClientRequestBuildFromRpcMsg(&clientRequestMsg, pMsg, 123, true, 1000); + SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(clientRequestMsg.contLen); + memcpy(pMsg2->data, clientRequestMsg.pCont, clientRequestMsg.contLen); + return pMsg2; } int main(int argc, char **argv) { @@ -207,8 +210,8 @@ int main(int argc, char **argv) { for (int i = 0; i < 10; ++i) { SyncClientRequest *pSyncClientRequest = pMsg1; SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); + // syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + // gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index bbf14c604e..6a8404308b 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index adebfe1be2..d5e7d71030 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -6,6 +6,8 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "syncTest.h" + void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 7c5334f668..2e5c26719b 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -7,6 +7,7 @@ #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" @@ -140,7 +141,8 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild(pMsg, 123, true, 1000); + SyncClientRequest *pRetMsg = NULL; + // syncClientRequestBuild(pMsg, 123, true, 1000); return pRetMsg; } @@ -179,14 +181,14 @@ int main(int argc, char **argv) { SyncClientRequest *pMsg1 = step1(pMsg0); syncClientRequestLog2((char *)"==step1==", pMsg1); - for (int i = 0; i < 10; ++i) { - SyncClientRequest *pSyncClientRequest = pMsg1; - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); + // for (int i = 0; i < 10; ++i) { + // SyncClientRequest *pSyncClientRequest = pMsg1; + // SRpcMsg rpcMsg = {0}; + // syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + // gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); - taosMsleep(1000); - } + // taosMsleep(1000); + // } while (1) { sTrace("while 1 sleep"); diff --git a/source/libs/sync/test/sync_test_lib/CMakeLists.txt b/source/libs/sync/test/sync_test_lib/CMakeLists.txt new file mode 100644 index 0000000000..db775c3662 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src SYNC_TEST_SRC) +add_library(sync_test_lib STATIC ${SYNC_TEST_SRC}) + +target_link_libraries( + sync_test_lib + PUBLIC sync +) + +target_include_directories( + sync_test_lib + PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/test/sync_test_lib/inc/syncIO.h similarity index 95% rename from source/libs/sync/inc/syncIO.h rename to source/libs/sync/test/sync_test_lib/inc/syncIO.h index cfc4dd2472..955a832b68 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncIO.h @@ -50,7 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); - int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg, SyncIndex *pRetIndex); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SRpcMsg *pMsg, SyncIndex *pRetIndex); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h new file mode 100644 index 0000000000..3a49074272 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_TEST_H +#define _TD_LIBS_SYNC_TEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "syncInt.h" + +#include "wal.h" + +#include "syncEnv.h" +#include "syncIO.h" +#include "syncIndexMgr.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftCfg.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncRespMgr.h" +#include "syncSnapshot.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); +char* syncEntry2Str(const SSyncRaftEntry* pEntry); +void syncEntryPrint(const SSyncRaftEntry* pObj); +void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); +void syncEntryLog(const SSyncRaftEntry* pObj); +void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); + +cJSON* raftCache2Json(SRaftEntryHashCache* pObj); +char* raftCache2Str(SRaftEntryHashCache* pObj); +void raftCachePrint(SRaftEntryHashCache* pObj); +void raftCachePrint2(char* s, SRaftEntryHashCache* pObj); +void raftCacheLog(SRaftEntryHashCache* pObj); +void raftCacheLog2(char* s, SRaftEntryHashCache* pObj); + +cJSON* raftEntryCache2Json(SRaftEntryCache* pObj); +char* raftEntryCache2Str(SRaftEntryCache* pObj); +void raftEntryCachePrint(SRaftEntryCache* pObj); +void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj); +void raftEntryCacheLog(SRaftEntryCache* pObj); +void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj); + +cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg); +char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg); + +cJSON* logStore2Json(SSyncLogStore* pLogStore); +char* logStore2Str(SSyncLogStore* pLogStore); +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); +char* logStoreSimple2Str(SSyncLogStore* pLogStore); +void logStorePrint(SSyncLogStore* pLogStore); +void logStorePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreLog(SSyncLogStore* pLogStore); +void logStoreLog2(char* s, SSyncLogStore* pLogStore); +void logStoreSimplePrint(SSyncLogStore* pLogStore); +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreSimpleLog(SSyncLogStore* pLogStore); +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); + +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c similarity index 98% rename from source/libs/sync/src/syncIO.c rename to source/libs/sync/test/sync_test_lib/src/syncIO.c index afa2d43e13..14adc18c66 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -279,10 +279,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { if (io->FpOnSyncClientRequest != NULL) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); - ASSERT(pSyncMsg != NULL); - io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); + io->FpOnSyncClientRequest(io->pSyncNode, pRpcMsg, NULL); } } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { diff --git a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c new file mode 100644 index 0000000000..99d124580b --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncNode2Json(const SSyncNode* pSyncNode) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pSyncNode != NULL) { + // init by SSyncInfo + cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); + cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); + cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); + cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncSendMSg); + cJSON_AddStringToObject(pRoot, "syncSendMSg", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); + cJSON_AddStringToObject(pRoot, "queue", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncEqMsg); + cJSON_AddStringToObject(pRoot, "syncEqMsg", u64buf); + + // init internal + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); + cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); + + cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); + cJSON* pPeers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); + } + cJSON* pPeersId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersId", pPeersId); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + } + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); + cJSON* pReplicasId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + } + + // raft algorithm + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); + cJSON_AddStringToObject(pRoot, "pFsm", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); + cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + + // life cycle + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + + // tla+ server vars + cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); + cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); + cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); + + // tla+ candidate vars + cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); + cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); + + // tla+ leader vars + cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); + cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); + + // tla+ log vars + cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + // timer ms init + cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); + cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); + cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); + + // ping timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); + cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); + cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); + cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); + + // elect timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); + cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); + cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); + cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); + cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); + + // heartbeat timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); + cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); + + // callback + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); + cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); + cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); + cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); + cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); + cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + + // restoreFinish + cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); + + // snapshot senders + cJSON* pSenders = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "senders", pSenders); + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); + } + + // snapshot receivers + cJSON* pReceivers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); + + // changing + cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); + return pJson; +} + +char* syncNode2Str(const SSyncNode* pSyncNode) { + cJSON* pJson = syncNode2Json(pSyncNode); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { + int32_t len = 256; + char* s = (char*)taosMemoryMalloc(len); + + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + snprintf(s, len, + "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", sby:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d", + pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, + logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); + + return s; +} + +// ping -------------- +int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + syncPingLog2((char*)"==syncNodePing==", pMsg); + int32_t ret = 0; + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); + + ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); + return ret; +} + +int32_t syncNodePingSelf(SSyncNode* pSyncNode) { + int32_t ret = 0; + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); + ASSERT(ret == 0); + + syncPingDestroy(pMsg); + return ret; +} + +int32_t syncNodePingPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* destId = &(pSyncNode->peersId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); + ASSERT(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +int32_t syncNodePingAll(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + SRaftId* destId = &(pSyncNode->replicasId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); + ASSERT(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c new file mode 100644 index 0000000000..012382d69d --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +// --------------------------------------------- +cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { + cJSON* pRoot; + + // in compiler optimization, switch case = if else constants + if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncTimeout2Json(pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { + SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncPing2Json(pSyncMsg); + syncPingDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncPingReply2Json(pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest* pSyncMsg = pRpcMsg->pCont; + pRoot = syncClientRequest2Json(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { + pRoot = syncRpcUnknownMsg2Json(); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncRequestVote2Json(pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncRequestVoteReply2Json(pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncAppendEntries2Json(pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncAppendEntriesReply2Json(pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotSend2Json(pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotRsp2Json(pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { + SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncLeaderTransfer2Json(pSyncMsg); + syncLeaderTransferDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + taosMemoryFree(s); + + } else { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + taosMemoryFree(s); + } + + cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); + cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); + cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); + // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); + return pJson; +} + +cJSON* syncRpcUnknownMsg2Json() { + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); + cJSON_AddStringToObject(pRoot, "data", "unknown message"); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); + return pJson; +} + +char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { + cJSON* pJson = syncRpcMsg2Json(pRpcMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncRpcMsgPrint(SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + printf("syncRpcMsgPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + printf("syncRpcMsgPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncRpcMsgLog(SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); + cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + + int32_t metaArrayLen = sizeof(SOffsetAndContLen) * pMsg->dataCount; // + int32_t entryArrayLen = pMsg->dataLen - metaArrayLen; + + cJSON_AddNumberToObject(pRoot, "metaArrayLen", metaArrayLen); + cJSON_AddNumberToObject(pRoot, "entryArrayLen", entryArrayLen); + + SOffsetAndContLen* metaArr = (SOffsetAndContLen*)(pMsg->data); + + cJSON* pMetaArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pMeta = cJSON_CreateObject(); + cJSON_AddNumberToObject(pMeta, "offset", metaArr[i].offset); + cJSON_AddNumberToObject(pMeta, "contLen", metaArr[i].contLen); + cJSON_AddItemToArray(pMetaArr, pMeta); + } + + cJSON* pEntryArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entryArr", pEntryArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)(pMsg->data + metaArr[i].offset); + cJSON* pEntryJson = syncEntry2Json(pEntry); + cJSON_AddItemToArray(pEntryArr, pEntryJson); + } + + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntriesBatch", pRoot); + return pJson; +} + +char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { + cJSON* pJson = syncAppendEntriesBatch2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + printf("syncAppendEntriesBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + printf("syncAppendEntriesBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + sTrace("syncAppendEntriesBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + sLTrace("syncAppendEntriesBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); + cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); + cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); + return pJson; +} + +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { + cJSON* pJson = syncSnapshotSend2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncClientRequestBatch2Json(const SyncClientRequestBatch* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); + + SRaftMeta* metaArr = syncClientRequestBatchMetaArr(pMsg); + SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pMsg); + + cJSON* pMetaArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pMeta = cJSON_CreateObject(); + cJSON_AddNumberToObject(pMeta, "seqNum", metaArr[i].seqNum); + cJSON_AddNumberToObject(pMeta, "isWeak", metaArr[i].isWeak); + cJSON_AddItemToArray(pMetaArr, pMeta); + } + + cJSON* pMsgArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "msgArr", pMsgArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pRpcMsgJson = syncRpcMsg2Json(&msgArr[i]); + cJSON_AddItemToArray(pMsgArr, pRpcMsgJson); + } + + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncClientRequestBatch", pRoot); + return pJson; +} + +char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { + cJSON* pJson = syncClientRequestBatch2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + printf("syncClientRequestBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + printf("syncClientRequestBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + sTrace("syncClientRequestBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncClientRequestBatch2Str(pMsg); + sLTrace("syncClientRequestBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// for debug ---------------------- +void syncClientRequestPrint(const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + printf("syncClientRequestPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + printf("syncClientRequestPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestLog(const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c new file mode 100644 index 0000000000..88b697fbf1 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + + diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c new file mode 100644 index 0000000000..8179b24d29 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pEntry != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); + cJSON_AddStringToObject(pRoot, "index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); + + char* s; + s = syncUtilPrintBin((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + + s = syncUtilPrintBin2((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); + return pJson; +} + +char* syncEntry2Str(const SSyncRaftEntry* pEntry) { + cJSON* pJson = syncEntry2Json(pEntry); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncEntryPrint(const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncEntryLog(const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryHashCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&pCache->mutex); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&pCache->mutex); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryHashCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryHashCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryHashCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sLTrace("raftCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* raftEntryCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&pCache->mutex); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); + while (tSkipListIterNext(pIter)) { + SSkipListNode* pNode = tSkipListIterGet(pIter); + ASSERT(pNode != NULL); + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + tSkipListDestroyIter(pIter); + + taosThreadMutexUnlock(&pCache->mutex); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftEntryCache2Str(SRaftEntryCache* pObj) { + cJSON* pJson = raftEntryCache2Json(pObj); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftEntryCachePrint(SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + printf("raftEntryCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + printf("raftEntryCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftEntryCacheLog(SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + sTrace("raftEntryCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj) { + if (gRaftDetailLog) { + char* serialized = raftEntryCache2Str(pObj); + sLTrace("raftEntryCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c new file mode 100644 index 0000000000..53e414ccfd --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* logStore2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + + if (!raftLogIsEmpty(pLogStore)) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + SSyncRaftEntry* pEntry = NULL; + raftLogGetEntry(pLogStore, i, &pEntry); + + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + } + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); + return pJson; +} + +char* logStore2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStore2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); + return pJson; +} + +char* logStoreSimple2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStoreSimple2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ----------------- +void logStorePrint(SSyncLogStore* pLogStore) { + char* serialized = logStore2Str(pLogStore); + printf("logStorePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStorePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStore2Str(pLogStore); + printf("logStorePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreLog(SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sLTrace("logStoreLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); + } +} + +void logStoreLog2(char* s, SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sLTrace("logStoreLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// for debug ----------------- +void logStoreSimplePrint(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} From a9167beb91be8a50154b6e0d9909a9f2635a9ea7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 14:52:37 +0800 Subject: [PATCH 11/56] fix(query): expand buffer size. --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a03c04ed6e..e3714be594 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -586,7 +586,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; + char tmpBuf[128] = {0}; int32_t tmpBufLen = 0; if (pLogBuf == NULL || pLogBuf->stop) return -1; From 65c813f735d0c8b98975bacfac97bf8de5fdce14 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 14:59:09 +0800 Subject: [PATCH 12/56] fix(query): fix some syntax errors. --- source/util/src/tlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e3714be594..a3bfd6e3d2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -317,10 +317,10 @@ static void taosGetLogFileName(char *fn) { for (int32_t i = 0; i < tsLogObj.fileNum; i++) { char fileName[LOG_FILE_NAME_LEN]; - sprintf(fileName, "%s%d.0", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN,"%s%d.0", fn, i); bool file1open = taosCheckFileIsOpen(fileName); - sprintf(fileName, "%s%d.1", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN,"%s%d.1", fn, i); bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { From 8659382c0eecafe3af30c9ed2fa1f38c40f957b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:11:52 +0800 Subject: [PATCH 13/56] refactor: update cmake file. --- cmake/cmake.version | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 92f7f8b895..d0a83155be 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -65,13 +65,14 @@ ELSE () ENDIF () MESSAGE(STATUS "============= compile version parameter information start ============= ") -MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) -MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) -MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) MESSAGE(STATUS "build date:" ${TD_VER_DATE}) -MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) -MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) -MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "build type:" ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) MESSAGE(STATUS "============= compile version parameter information end ============= ") STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) From f942e319b92d4545226321fca22caf3b72ff9c24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:15:35 +0800 Subject: [PATCH 14/56] fix(query): fix some syntax errors. --- cmake/cmake.version | 16 ++++++++-------- source/util/src/tlog.c | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index d0a83155be..03598519ed 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -65,14 +65,14 @@ ELSE () ENDIF () MESSAGE(STATUS "============= compile version parameter information start ============= ") -MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) -MESSAGE(STATUS "compatible:" ${TD_VER_COMPATIBLE}) -MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) -MESSAGE(STATUS "build date:" ${TD_VER_DATE}) -MESSAGE(STATUS "build type:" ${CMAKE_BUILD_TYPE}) -MESSAGE(STATUS "type:" ${TD_VER_VERTYPE}) -MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) -MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) +MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) +MESSAGE(STATUS "build date: " ${TD_VER_DATE}) +MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "type: " ${TD_VER_VERTYPE}) +MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) MESSAGE(STATUS "============= compile version parameter information end ============= ") STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a3bfd6e3d2..c72be76a95 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -324,7 +324,7 @@ static void taosGetLogFileName(char *fn) { bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { - sprintf(tsLogObj.logName, "%s%d", fn, i); + snprintf(tsLogObj.logName, tListLen(tsLogObj.logName), "%s%d", fn, i); return; } } From 2ee0cf6701b4b62648ff70bd99de44921e7ab5a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:19:27 +0800 Subject: [PATCH 15/56] fix(query): fix the string copy error. --- source/libs/function/src/tudf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 1dedfe8364..8715aa0be1 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -131,7 +131,8 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char udfdPathLdLib[1024] = {0}; size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); - strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); + strncpy(udfdPathLdLib, tsUdfdLdLibPath, tListLen(udfdPathLdLib)); + udfdPathLdLib[udfdLdLibPathLen] = ':'; strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { From 683e2cff7c08393804bd4dd3c4e0d0d6331b7179 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 15:49:39 +0800 Subject: [PATCH 16/56] change test cases --- tests/script/tsim/db/alter_option.sim | 4 +-- tests/script/tsim/db/create_all_options.sim | 4 +-- tests/system-test/1-insert/alter_database.py | 29 ++++++++++++-------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index b6139cea13..63018aea8c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -38,7 +38,7 @@ endi print ============= create database #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both'] # | WAL_FSYNC_PERIOD value [0 ~ 180000 ms] @@ -78,7 +78,7 @@ endi if $data7_db != 1440000m,1440000m,1440000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 5ac2ee6c4e..7012fbac6c 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -37,7 +37,7 @@ endi print ============= create database with all options #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | PAGESIZE value [1~16384, default: 4] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node'] @@ -98,7 +98,7 @@ endi if $data7_db != 5256000m,5256000m,5256000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 1918c0ef76..aec18a1cf1 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -10,37 +10,43 @@ from util.sql import * from util.cases import * from util.dnodes import * + class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),logSql) - self.buffer_boundary = [3,4097,8193,12289,16384] - self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + tdSql.init(conn.cursor(), logSql) + self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + self.buffer_error = [self.buffer_boundary[0] - + 1, self.buffer_boundary[-1]+1, 12289, 256] # pages_boundary >= 64 - self.pages_boundary = [64,128,512] + self.pages_boundary = [64, 128, 512] self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): tdSql.execute('create database db') for buffer in self.buffer_boundary: tdSql.execute(f'alter database db buffer {buffer}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8], buffer) tdSql.execute('drop database db') tdSql.execute('create database db vgroups 10') for buffer in self.buffer_error: tdSql.error(f'alter database db buffer {buffer}') tdSql.execute('drop database db') - + def alter_pages(self): tdSql.execute('create database db') for pages in self.pages_boundary: tdSql.execute(f'alter database db pages {pages}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10], pages) tdSql.execute('drop database db') tdSql.execute('create database db') - tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') self.pages_error.append(tdSql.queryResult[0][10]) for pages in self.pages_error: tdSql.error(f'alter database db pages {pages}') @@ -55,5 +61,6 @@ class TDTestCase: tdSql.close() tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From fa45cd2ce89f0214f17c157d7105d30a7b36cee8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:50:31 +0800 Subject: [PATCH 17/56] enh(query): unfold the loop. --- source/libs/function/src/builtinsimpl.c | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 47977f9757..fbcaac95df 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3113,6 +3113,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } } else { +#if 0 for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; @@ -3127,6 +3128,64 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { pResInfo->numOfRes = 1; } } +#else + + if (!pInputCol->hasNull) { + int32_t round = pInput->numOfRows >> 2; + int32_t reminder = pInput->numOfRows & 0x03; + + int32_t tick = 0; + for (int32_t i = pInput->startRowIndex; tick < round; i += 4, tick += 1) { + int64_t cts = pts[i]; + int32_t chosen = i; + + if (cts < pts[i + 1]) { + cts = pts[i + 1]; + chosen = i + 1; + } + + if (cts < pts[i + 2]) { + cts = pts[i + 2]; + chosen = i + 2; + } + + if (cts < pts[i + 3]) { + cts = pts[i + 3]; + chosen = i + 3; + } + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + char* data = colDataGetNumData(pInputCol, chosen); + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } + + for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + numOfElems++; + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + char* data = colDataGetNumData(pInputCol, i); + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetNumData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } + } +#endif } #endif From 86ed5e5674d28da33c3eea30466acc544412e324 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 16:02:04 +0800 Subject: [PATCH 18/56] more optimize --- source/dnode/vnode/src/inc/tsdb.h | 44 +++++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 25 ------------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 +----- source/dnode/vnode/src/vnd/vnodeModule.c | 6 +++ 4 files changed, 45 insertions(+), 42 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a5257b32c0..ba9d68ee7f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -110,7 +110,6 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { #define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // SRowIter void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); @@ -210,11 +209,10 @@ void tsdbRefMemTable(SMemTable *pMemTable); void tsdbUnrefMemTable(SMemTable *pMemTable); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter -int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); -void *tsdbTbDataIterDestroy(STbDataIter *pIter); -void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter); -bool tsdbTbDataIterNext(STbDataIter *pIter); +int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); +void *tsdbTbDataIterDestroy(STbDataIter *pIter); +void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); +bool tsdbTbDataIterNext(STbDataIter *pIter); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== @@ -772,6 +770,40 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +static FORCE_INLINE int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = tGetI64(p, &pRow->version); + pRow->pTSRow = (STSRow *)(p + n); + n += pRow->pTSRow->len; + return n; +} + +static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { + if (pIter == NULL) return NULL; + + if (pIter->pRow) { + return pIter->pRow; + } + + if (pIter->backward) { + if (pIter->pNode == pIter->pTbData->sl.pHead) { + return NULL; + } + } else { + if (pIter->pNode == pIter->pTbData->sl.pTail) { + return NULL; + } + } + + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + pIter->pRow = &pIter->row; + + return pIter->pRow; +} + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index c663e2b526..52c7a07c49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -294,31 +294,6 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { - // we add here for commit usage - if (pIter == NULL) return NULL; - - if (pIter->pRow) { - goto _exit; - } - - if (pIter->backward) { - if (pIter->pNode == pIter->pTbData->sl.pHead) { - goto _exit; - } - } else { - if (pIter->pNode == pIter->pTbData->sl.pTail) { - goto _exit; - } - } - - tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); - pIter->pRow = &pIter->row; - -_exit: - return pIter->pRow; -} - static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 52b74aea3f..9ac9a7d66d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -575,16 +575,6 @@ int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { return n; } -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { - int32_t n = 0; - - n += tGetI64(p, &pRow->version); - pRow->pTSRow = (STSRow *)(p + n); - n += pRow->pTSRow->len; - - return n; -} - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } @@ -1053,7 +1043,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS tRowIterInit(&rIter, pRow, pTSchema); pColVal = tRowIterNext(&rIter); for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; while (pColVal && pColVal->cid < pColData->cid) { pColVal = tRowIterNext(&rIter); diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 9fe37505ad..99d6decfc9 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -37,6 +37,12 @@ struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); +static sem_t canCommit = {0}; + +static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; +void vnode_wait_commit() { tsem_wait(&canCommit); } +void vnode_done_commit() { tsem_wait(&canCommit); } + int vnodeInit(int nthreads) { int8_t init; int ret; From 652c6388f2aeb5a52849ef6a3af3264b7b672b8c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 9 Nov 2022 16:40:17 +0800 Subject: [PATCH 19/56] fix: fix valgrind invalid read error --- source/libs/scalar/src/sclfunc.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index c89072233f..22fa99d75d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1028,11 +1028,11 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t type = GET_PARAM_TYPE(pInput); bool tzPresent = (inputNum == 2) ? true : false; - char *tz; - int32_t tzLen; + char tz[20] = {0}; + int32_t tzLen = 0; if (tzPresent) { - tz = varDataVal(pInput[1].columnData->pData); tzLen = varDataLen(pInput[1].columnData->pData); + memcpy(tz, varDataVal(pInput[1].columnData->pData), tzLen); } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -1071,8 +1071,10 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t len = (int32_t)strlen(buf); // add timezone string - snprintf(buf + len, tzLen + 1, "%s", tz); - len += tzLen; + if (tzLen > 0) { + snprintf(buf + len, tzLen + 1, "%s", tz); + len += tzLen; + } if (hasFraction) { int32_t fracLen = (int32_t)strlen(fraction) + 1; From 3f62f14cee3a952c9155a235991e69fc5e1e7797 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 16:54:33 +0800 Subject: [PATCH 20/56] refactor: do some internal refactor. --- source/libs/function/src/builtinsimpl.c | 38 ++++++------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fbcaac95df..55642fa1a1 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3097,22 +3097,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { #else int64_t* pts = (int64_t*)pInput->pPTS->pData; - if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; - } - - numOfElems++; - - char* data = colDataGetVarData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); - pResInfo->numOfRes = 1; - } - } - } else { #if 0 for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { @@ -3120,16 +3104,13 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - - char* data = colDataGetNumData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); pResInfo->numOfRes = 1; } } #else - if (!pInputCol->hasNull) { int32_t round = pInput->numOfRows >> 2; int32_t reminder = pInput->numOfRows & 0x03; @@ -3155,7 +3136,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - char* data = colDataGetNumData(pInputCol, chosen); + char* data = colDataGetData(pInputCol, chosen); doSaveCurrentVal(pCtx, i, cts, type, data); pResInfo->numOfRes = 1; } @@ -3163,10 +3144,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { numOfElems++; - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - char* data = colDataGetNumData(pInputCol, i); - doSaveCurrentVal(pCtx, i, cts, type, data); + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); pResInfo->numOfRes = 1; } } @@ -3179,14 +3159,14 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { - char* data = colDataGetNumData(pInputCol, i); + char* data = colDataGetData(pInputCol, i); doSaveCurrentVal(pCtx, i, pts[i], type, data); pResInfo->numOfRes = 1; } } } #endif - } + #endif // save selectivity value for column consisted of all null values From f4a5cb995d889b9cae32fc536ddebc939250ab95 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 9 Nov 2022 17:04:22 +0800 Subject: [PATCH 21/56] Revert "test:refact unsuccessful case" This reverts commit d550e43c5aaf535de40534f5954ee1570536f43b. --- tests/system-test/fulltest.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4228d1e419..0729d36f12 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -76,8 +76,8 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/count_partition.py -R python3 ./test.py -f 2-query/count.py python3 ./test.py -f 2-query/count.py -R -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/db.py -R python3 ./test.py -f 2-query/diff.py @@ -393,7 +393,7 @@ python3 ./test.py -f 2-query/max.py -Q 2 python3 ./test.py -f 2-query/min.py -Q 2 python3 ./test.py -f 2-query/mode.py -Q 2 python3 ./test.py -f 2-query/count.py -Q 2 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 python3 ./test.py -f 2-query/last.py -Q 2 python3 ./test.py -f 2-query/first.py -Q 2 python3 ./test.py -f 2-query/To_iso8601.py -Q 2 @@ -490,7 +490,7 @@ python3 ./test.py -f 2-query/max.py -Q 3 python3 ./test.py -f 2-query/min.py -Q 3 python3 ./test.py -f 2-query/mode.py -Q 3 python3 ./test.py -f 2-query/count.py -Q 3 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 python3 ./test.py -f 2-query/last.py -Q 3 python3 ./test.py -f 2-query/first.py -Q 3 python3 ./test.py -f 2-query/To_iso8601.py -Q 3 @@ -589,7 +589,7 @@ python3 ./test.py -f 2-query/max.py -Q 4 python3 ./test.py -f 2-query/min.py -Q 4 python3 ./test.py -f 2-query/mode.py -Q 4 python3 ./test.py -f 2-query/count.py -Q 4 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 python3 ./test.py -f 2-query/last.py -Q 4 python3 ./test.py -f 2-query/first.py -Q 4 python3 ./test.py -f 2-query/To_iso8601.py -Q 4 From fcc50467f43f20abd303bd80fbc37d6af23b6ff2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 17:33:22 +0800 Subject: [PATCH 22/56] fix(sync): fix AddressSanitizer error --- source/libs/sync/src/syncMain.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f6925946e9..85668068a5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1040,6 +1040,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); + pSyncNode->pRaftStore = NULL; syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -2174,10 +2175,18 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = pData->pSyncNode; SSyncTimer* pSyncTimer = pData->pTimer; + if (pSyncNode == NULL) { + return; + } + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { return; } + if (pSyncNode->pRaftStore == NULL) { + return; + } + // sNTrace(pSyncNode, "eq peer hb timer"); int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); From f2fd847414f7841eec4e792fe61ee3063265f8d2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 17:35:22 +0800 Subject: [PATCH 23/56] refact: move sync test code to single lib --- source/libs/sync/src/syncMain.c | 8 +------- source/libs/sync/src/syncReplication.c | 2 +- tests/script/tsim/vnode/replica3_repeat.sim | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d05e1608e2..b4ca3f5b01 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2223,10 +2223,6 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn } else { // del resp mgr, call FpCommitCb - - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SFsmCbMeta cbMeta = { .index = pEntry->index, .lastConfigIndex = SYNC_INDEX_INVALID, @@ -2238,9 +2234,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn .currentTerm = ths->pRaftStore->currentTerm, .flag = 0, }; - - syncRespMgrGetAndDel(ths->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, &cbMeta); + ths->pFsm->FpCommitCb(ths->pFsm, pMsg, &cbMeta); if (h) { taosLRUCacheRelease(ths->pLogStore->pCache, h, false); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 8c3f7e5384..3f22a1f2cd 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -78,7 +78,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) { pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); ASSERT(pMsg != NULL); - memcpy(pMsg->data, pEntry->data, pEntry->bytes); + memcpy(pMsg->data, pEntry, pEntry->bytes); syncEntryDestory(pEntry); } else { diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index ccff06819b..8efba515ae 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -118,7 +118,7 @@ step7: sql select count(*) from db.tb -x step7 print select count(*) from db.tb ==> $data00 $lastRows -if $data00 <= $lastRows then +if $data00 < $lastRows then return -1 endi From 153c01ee7328678101f434edf896a66dbdcec90c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 17:38:46 +0800 Subject: [PATCH 24/56] fix(sync): fix AddressSanitizer error --- source/libs/sync/src/syncMain.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b4a3d30f7d..09be3e45c3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1040,6 +1040,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); + pSyncNode->pRaftStore = NULL; syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -2174,10 +2175,18 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = pData->pSyncNode; SSyncTimer* pSyncTimer = pData->pTimer; + if (pSyncNode == NULL) { + return; + } + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { return; } + if (pSyncNode->pRaftStore == NULL) { + return; + } + // sNTrace(pSyncNode, "eq peer hb timer"); int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); From 6298f92d563f768909d3abde1b4f864b5bdeef7d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Nov 2022 17:39:28 +0800 Subject: [PATCH 25/56] fix: some problems of parser --- source/libs/parser/src/parTranslater.c | 34 +++++++++++++++++++++-- source/libs/parser/test/parSelectTest.cpp | 2 ++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dea3b959ed..82170a27d8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2393,6 +2393,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { pCxt->stableQuery = true; } + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isSelectStmt(pCxt->pCurrStmt)) { + ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; + } code = addNamespace(pCxt, pRealTable); } break; @@ -3428,6 +3431,19 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return (SNode*)pCol; } +// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail +static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type != r->type) { + return 1; + } + + if (l->bytes != r->bytes) { + return l->bytes > r->bytes ? 1 : -1; + } + + return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; +} + static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) { SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); @@ -3440,7 +3456,8 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS FORBOTH(pLeft, pLeftProjections, pRight, pRightProjections) { SExprNode* pLeftExpr = (SExprNode*)pLeft; SExprNode* pRightExpr = (SExprNode*)pRight; - if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { + int32_t comp = dataTypeComp(&pLeftExpr->resType, &pRightExpr->resType); + if (comp > 0) { SNode* pRightFunc = NULL; int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); if (TSDB_CODE_SUCCESS != code) { @@ -3448,9 +3465,20 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } REPLACE_LIST2_NODE(pRightFunc); pRightExpr = (SExprNode*)pRightFunc; + } else if (comp < 0) { + SNode* pLeftFunc = NULL; + int32_t code = createCastFunc(pCxt, pLeft, pRightExpr->resType, &pLeftFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_LIST1_NODE(pLeftFunc); + SExprNode* pLeftFuncExpr = (SExprNode*)pLeftFunc; + snprintf(pLeftFuncExpr->aliasName, sizeof(pLeftFuncExpr->aliasName), "%s", pLeftExpr->aliasName); + snprintf(pLeftFuncExpr->userAlias, sizeof(pLeftFuncExpr->userAlias), "%s", pLeftExpr->userAlias); + pLeft = pLeftFunc; + pLeftExpr = pLeftFuncExpr; } - strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); - pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; + snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pSetOperator->stmtName, pLeft))) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 0027ac9ca1..fcd8dd1f26 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -425,6 +425,8 @@ TEST_F(ParserSelectTest, informationSchema) { run("SELECT * FROM ins_databases WHERE name = 'information_schema'"); run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'"); + + run("SELECT * FROM (SELECT table_name FROM ins_tables) t WHERE table_name = 'a'"); } TEST_F(ParserSelectTest, withoutFrom) { From 9f7e06721c3009f1f57dabc2462f7b0aa752809d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 9 Nov 2022 18:48:52 +0800 Subject: [PATCH 26/56] enh: support vgroup epset update --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 3 ++- include/libs/qcom/query.h | 1 + source/client/src/clientHb.c | 4 +++- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 27 +++++++++++++++++---- source/libs/catalog/src/catalog.c | 4 ++-- source/libs/catalog/src/ctgAsync.c | 1 + source/libs/catalog/src/ctgCache.c | 29 +++++++++++++---------- source/libs/catalog/src/ctgDbg.c | 26 +++++++++++++++++--- source/libs/catalog/test/catalogTests.cpp | 3 ++- source/libs/parser/inc/parUtil.h | 2 +- source/libs/parser/src/parTranslater.c | 8 +++---- source/libs/parser/src/parUtil.c | 3 ++- source/libs/parser/test/mockCatalog.cpp | 2 +- source/libs/qcom/src/querymsg.c | 4 +++- 16 files changed, 86 insertions(+), 34 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b840053b08..75c34f88a2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -190,6 +190,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; } SBuildUseDBInput; typedef struct SField { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 7717d3bccc..6154882756 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -56,6 +56,7 @@ typedef struct SDbInfo { int32_t vgVer; int32_t tbNum; int64_t dbId; + int64_t stateTs; } SDbInfo; typedef struct STablesReq { @@ -153,7 +154,7 @@ int32_t catalogInit(SCatalogCfg* cfg); */ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* stateTs); /** * Get a DB's all vgroup info. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 74c1a8c07c..40b7e97fc7 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -127,6 +127,7 @@ typedef struct SDBVgInfo { int16_t hashSuffix; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo } SDBVgInfo; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index fa0e3212a3..bf34d3e2df 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -61,7 +61,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->uid); + tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, rsp->uid); if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); @@ -72,6 +72,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } vgInfo->vgVersion = rsp->vgVersion; + vgInfo->stateTs = rsp->stateTs; vgInfo->hashMethod = rsp->hashMethod; vgInfo->hashPrefix = rsp->hashPrefix; vgInfo->hashSuffix = rsp->hashSuffix; @@ -486,6 +487,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); db->numOfTable = htonl(db->numOfTable); + db->stateTs = htobe64(db->stateTs); } SKv kv = { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e17e7e79c6..095bc2d709 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -181,7 +181,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); struct SCatalog* pCatalog = NULL; - if (usedbRsp.vgVersion >= 0) { + if (usedbRsp.vgVersion >= 0) { // cached in local uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f2a4462bf5..9973f02bb9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1175,7 +1175,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { + if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || pReq->stateTs < pDb->stateTs) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); } @@ -1275,12 +1275,31 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, for (int32_t i = 0; i < numOfDbs; ++i) { SDbVgVersion *pDbVgVersion = &pDbs[i]; - pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->dbId = be64toh(pDbVgVersion->dbId); pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable); + pDbVgVersion->stateTs = be64toh(pDbVgVersion->stateTs); SUseDbRsp usedbRsp = {0}; + if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); + if (pDbVgVersion->vgVersion < vgVersion) { + usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); + + mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); + usedbRsp.vgVersion = vgVersion++; + } else { + usedbRsp.vgVersion = pDbVgVersion->vgVersion; + } + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + + continue; + } + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); if (pDb == NULL) { mTrace("db:%s, no exist", pDbVgVersion->dbFName); @@ -1293,8 +1312,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable /* && - pDbVgVersion->stateTs == pDb->stateTs */) { + if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && + pDbVgVersion->stateTs == pDb->stateTs) { mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1f87066c82..2dcd681205 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -715,10 +715,10 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* pStateTs) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { + if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId || NULL == tableNum || NULL == pStateTs) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index b601865306..ff3bd4d33c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1998,6 +1998,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; + pInfo->stateTs = dbCache->vgCache.vgInfo->stateTs; ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 51807a145a..e99beca69a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1231,14 +1231,16 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_STAT_INC(numOfDb, 1); - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0}; tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + if (!IS_SYS_DBNAME(dbFName)) { + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + } return TSDB_CODE_SUCCESS; @@ -1563,7 +1565,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); @@ -1579,15 +1581,15 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SDBVgInfo *vgInfo = vgCache->vgInfo; if (dbInfo->vgVersion < vgInfo->vgVersion) { - ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgDebug("db updateVgroup is ignored, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgWUnlockVgInfo(dbCache); goto _return; } - if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { - ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, - dbInfo->numOfTable); + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable && dbInfo->stateTs == vgInfo->stateTs) { + ctgDebug("no new db vgroup update info, dbFName:%s, vgVer:%d, numOfTable:%d, stateTs:%" PRId64, dbFName, dbInfo->vgVersion, + dbInfo->numOfTable, dbInfo->stateTs); ctgWUnlockVgInfo(dbCache); goto _return; @@ -1599,15 +1601,17 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, stateTs:%" PRId64 ", dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.stateTs, vgVersion.dbId); ctgWUnlockVgInfo(dbCache); dbCache = NULL; - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + if (!IS_SYS_DBNAME(dbFName)) { + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + } _return: @@ -2170,7 +2174,6 @@ void *ctgUpdateThreadFunc(void *param) { CTG_RT_STAT_INC(numOfOpDequeue, 1); ctgdShowCacheInfo(); - ctgdShowClusterCache(pCtg); } qInfo("catalog update thread stopped"); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 26b5903bb0..b6ff1c8b98 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -78,7 +78,7 @@ void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { num = taosArrayGetSize(pResult->pDbInfo); for (int32_t i = 0; i < num; ++i) { SDbInfo *pDb = taosArrayGet(pResult->pDbInfo, i); - qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId); + qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, stateTs:%" PRId64 " dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->stateTs, pDb->dbId); } } else { qDebug("empty db info"); @@ -462,6 +462,7 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { int32_t hashMethod = -1; int16_t hashPrefix = 0; int16_t hashSuffix = 0; + int64_t stateTs = 0; int32_t vgNum = 0; if (dbCache->vgCache.vgInfo) { @@ -469,16 +470,35 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { hashMethod = dbCache->vgCache.vgInfo->hashMethod; hashPrefix = dbCache->vgCache.vgInfo->hashPrefix; hashSuffix = dbCache->vgCache.vgInfo->hashSuffix; + stateTs = dbCache->vgCache.vgInfo->stateTs; if (dbCache->vgCache.vgInfo->vgHash) { vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } ctgDebug("[%d] db [%.*s][0x%" PRIx64 - "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, + "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, stateTs, hashMethod, hashPrefix, hashSuffix, vgNum); + if (dbCache->vgCache.vgInfo) { + int32_t i = 0; + void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL); + while (pVgIter) { + SVgroupInfo * pVg = (SVgroupInfo *)pVgIter; + + ctgDebug("The %04dth VG [id:%d, hashBegin:%u, hashEnd:%u, numOfTable:%d, epNum:%d, inUse:%d]", + i++, pVg->vgId, pVg->hashBegin, pVg->hashEnd, pVg->numOfTable, pVg->epSet.numOfEps, pVg->epSet.inUse); + + for (int32_t n = 0; n < pVg->epSet.numOfEps; ++n) { + SEp *pEp = &pVg->epSet.eps[n]; + ctgDebug("\tEp %d [fqdn:%s, port:%d]", n, pEp->fqdn, pEp->port); + } + + pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter); + } + } + pIter = taosHashIterate(dbHash, pIter); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index ebf7c7baeb..5e543384ac 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2486,7 +2486,8 @@ TEST(dbVgroup, getSetDbVgroupCase) { int32_t dbVer = 0; int64_t dbId = 0; int32_t tbNum = 0; - code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum); + int64_t stateTs = 0; + code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum, &stateTs); ASSERT_EQ(code, 0); ASSERT_EQ(dbVer, ctgTestVgVersion); ASSERT_EQ(dbId, ctgTestDbId); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 10a86866d5..c53d3f9320 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -108,7 +108,7 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum); + int32_t* pTableNum, int64_t* pStateTs); int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo); int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, bool* pPass); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dea3b959ed..dc43985a70 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -466,14 +466,14 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SParseContext* pParCxt = pCxt->pParseCxt; int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { if (pParCxt->async) { - code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum); + code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } else { - code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum); + code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } } if (TSDB_CODE_SUCCESS != code) { @@ -4941,7 +4941,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameExtractFullName(&name, usedbReq.db); - int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable); + int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 466c6edd24..00a72a1946 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -876,13 +876,14 @@ int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCac } int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SDbInfo* pDbInfo = NULL; int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo); if (TSDB_CODE_SUCCESS == code) { *pVersion = pDbInfo->vgVer; *pDbId = pDbInfo->dbId; *pTableNum = pDbInfo->tbNum; + *pStateTs = pDbInfo->stateTs; } return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 4f5ddd9a51..8f051c67a0 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -249,7 +249,7 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con } int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, - int32_t* tableNum) { + int32_t* tableNum, int64_t* stateTs) { return 0; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 181abc2247..fadc39f21d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -41,8 +41,9 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix; pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix; + pOut->dbVgroup->stateTs = usedbRsp->stateTs; - qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db); + qDebug("Got %d vgroup for db %s, vgVersion:%d, stateTs:%" PRId64, usedbRsp->vgNum, usedbRsp->db, usedbRsp->vgVersion, usedbRsp->stateTs); if (usedbRsp->vgNum <= 0) { return TSDB_CODE_SUCCESS; @@ -103,6 +104,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms usedbReq.vgVersion = pInput->vgVersion; usedbReq.dbId = pInput->dbId; usedbReq.numOfTable = pInput->numOfTable; + usedbReq.stateTs = pInput->stateTs; int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = (*mallcFp)(bufLen); From 0c92bd6e376bc747b12007c2742ccae4afdeea84 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 18:58:35 +0800 Subject: [PATCH 27/56] make pass ci --- source/dnode/vnode/src/vnd/vnodeModule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 99d6decfc9..782ffd788d 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -37,7 +37,7 @@ struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); -static sem_t canCommit = {0}; +static tsem_t canCommit = {0}; static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; void vnode_wait_commit() { tsem_wait(&canCommit); } From 98b77fe8b473b093b3136d29e414352c72c95529 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 19:02:47 +0800 Subject: [PATCH 28/56] refact: adjust head file --- source/libs/sync/inc/syncAppendEntries.h | 4 ---- source/libs/sync/inc/syncAppendEntriesReply.h | 4 ---- source/libs/sync/inc/syncCommit.h | 4 ---- source/libs/sync/inc/syncElection.h | 4 ---- source/libs/sync/inc/syncIndexMgr.h | 4 ---- source/libs/sync/inc/syncMessage.h | 5 ----- source/libs/sync/inc/syncRaftCfg.h | 5 ----- source/libs/sync/inc/syncRaftEntry.h | 5 ----- source/libs/sync/inc/syncRaftLog.h | 4 ---- source/libs/sync/inc/syncRaftStore.h | 5 ----- source/libs/sync/inc/syncReplication.h | 4 ---- source/libs/sync/inc/syncRequestVote.h | 4 ---- source/libs/sync/inc/syncRequestVoteReply.h | 4 ---- source/libs/sync/inc/syncRespMgr.h | 4 ---- source/libs/sync/inc/syncSnapshot.h | 5 ----- source/libs/sync/inc/syncTimeout.h | 4 ---- source/libs/sync/src/syncRaftEntry.c | 1 + 17 files changed, 1 insertion(+), 69 deletions(-) diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index dc40e2fc72..0a67939d35 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 0227d832fc..d2dff92d43 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h index 7458ce28ab..d3ba556f82 100644 --- a/source/libs/sync/inc/syncCommit.h +++ b/source/libs/sync/inc/syncCommit.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 9ccd9dd28f..662c17094e 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // RequestVote(i, j) == diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index e8f17537b4..bd88f5cdce 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 19079d518d..936081c7b2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" // --------------------------------------------- cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 15ca82664a..6b6ca21399 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define CONFIG_FILE_LEN 2048 diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 8b8ab41b53..2a73b95e4d 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -20,13 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" -#include "tref.h" #include "tskiplist.h" typedef struct SSyncRaftEntry { diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index c25d4ae34e..17ea36c640 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncRaftEntry.h" -#include "taosdef.h" #include "wal.h" typedef struct SSyncLogStoreData { diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index ac298c5375..077e6a7658 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define RAFT_STORE_BLOCK_SIZE 512 #define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 4f15a45cec..e08430327b 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // AppendEntries(i, j) == diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 73b4a0efae..c51ea4eea6 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 6bef18405c..1adab30aaa 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index 9026ecb66e..1d41c0a85f 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" typedef struct SRespStub { SRpcMsg rpcMsg; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 5594fe46ed..d688a3c1bd 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -20,13 +20,8 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" #define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 112a3d8610..e1a6050d92 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // Timeout(i) == /\ state[i] \in {Follower, Candidate} diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index c5b1399c8c..262a7b0879 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "syncRaftEntry.h" #include "syncUtil.h" +#include "tref.h" SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; From fb9e62afe24cdab219a71cc7ad9fac229c82e362 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 19:02:50 +0800 Subject: [PATCH 29/56] refactor(sync): if index less than wal.commitVer, do not truncate, otherwise it will print much log --- source/libs/sync/src/syncRaftLog.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d3d69b288b..7b1158319e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -301,6 +301,12 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return 0; } + // need not truncate + SyncIndex walCommitVer = walGetCommittedVer(pWal); + if (fromIndex <= walCommitVer) { + return 0; + } + int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; From 2402fd1d4867bbf5a67a87a8964eab5646b01beb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 19:14:27 +0800 Subject: [PATCH 30/56] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/cachescanoperator.c | 13 +- source/libs/executor/src/exchangeoperator.c | 11 +- source/libs/executor/src/executorimpl.c | 35 +++-- source/libs/executor/src/groupoperator.c | 27 +--- source/libs/executor/src/joinoperator.c | 8 +- source/libs/executor/src/projectoperator.c | 23 +--- source/libs/executor/src/scanoperator.c | 80 +++-------- source/libs/executor/src/sortoperator.c | 31 +---- source/libs/executor/src/tfill.c | 12 +- source/libs/executor/src/timewindowoperator.c | 129 ++++++------------ source/libs/function/src/builtinsimpl.c | 6 +- 12 files changed, 117 insertions(+), 262 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 77915624dd..62146b6048 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -881,7 +881,9 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul void doDestroyExchangeOperatorInfo(void* param); -void doSetOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, + SExecTaskInfo* pTaskInfo); void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo * pCache); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 41fcc44f97..6f9084ce52 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -93,12 +93,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - pOperator->name = "LastrowScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = @@ -126,7 +121,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); if (size == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -182,7 +177,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->indexOfBufferedRes += 1; return pRes; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } else { @@ -234,7 +229,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index cb3330e9ca..049de727df 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -213,7 +213,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { pExchangeInfo->limitInfo.numOfOutputRows += rows; if (rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } else { return pBlock; @@ -289,13 +289,8 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pOperator->name = "ExchangeOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); return pOperator; @@ -506,7 +501,7 @@ void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, pLoadInfo->totalElapsed / 1000.0); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ca1d4cccab..709e981a1f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -94,7 +94,7 @@ static void destroyIntervalOperatorInfo(void* param); static void destroyOperatorInfo(SOperatorInfo* pOperator); -void doSetOperatorCompleted(SOperatorInfo* pOperator) { +void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); @@ -102,6 +102,16 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, + void* pInfo, SExecTaskInfo* pTaskInfo) { + pOperator->name = (char*)name; + pOperator->operatorType = type; + pOperator->blocking = blocking; + pOperator->status = status; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; +} + int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; @@ -1795,7 +1805,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -1805,7 +1815,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2054,7 +2064,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { if (pInfo->totalInputRows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -2131,7 +2141,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { while (true) { fillResult = doFillImpl(pOperator); if (fillResult == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2361,13 +2371,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->groupId = UINT64_MAX; - pOperator->name = "TableAggregate"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_AGG; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); @@ -2564,14 +2569,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pOperator->name = "FillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; + setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index cfcdd0ee64..cf8dd110b6 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -316,7 +316,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -438,12 +438,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "GroupbyAggOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); @@ -654,7 +649,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { // try next group data ++pInfo->groupIndex; if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); clearPartitionOperator(pInfo); return NULL; } @@ -821,14 +816,9 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; + setOperatorInfo(pOperator, "PartitionOperator", QUERY_NODE_PHYSICAL_PLAN_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); @@ -963,7 +953,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->pInputDataBlock = NULL; SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } printDataBlock(pBlock, "stream partitionby recv"); @@ -1103,14 +1093,9 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols); - pOperator->name = "StreamPartitionOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; + setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index e7fe2bfc12..4e1daac643 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -73,14 +73,10 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; + + setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 58cc082653..ce1d13775c 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -98,12 +98,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys } pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, NULL); @@ -153,7 +149,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return PROJECT_RETRIEVE_DONE; } @@ -187,7 +183,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS // TODO: optimize it later when partition by + limit if ((pLimitInfo->slimit.limit == -1 && pLimitInfo->currentGroupId == 0) || (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -252,7 +248,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { @@ -400,12 +396,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->binfo.pRes = pResBlock; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - pOperator->name = "IndefinitOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - + setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -498,7 +489,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -627,7 +618,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 903843bcc4..3da54509ce 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -820,7 +820,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -843,7 +843,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -865,7 +865,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -947,13 +947,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; - pOperator->name = "TableScanOperator"; // for debug purpose - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024*128, -1, .5); if (pInfo->metaCache.pTableMetaEntryCache == NULL) { @@ -986,13 +981,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pInfo->dataReader = pReadHandle; // pInfo->prevGroupId = -1; - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1148,13 +1137,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - pOperator->name = "DataBlockDistScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; @@ -2367,9 +2350,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; - pOperator->name = "RawScanOperator"; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; @@ -2555,13 +2536,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->partitionSup.needCalc = false; - pOperator->name = "StreamScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamScanOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); @@ -2899,7 +2875,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2952,7 +2928,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3742,7 +3718,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } if (i >= taosArrayGetSize(pIdx->uids)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } else { pIdx->lastIdx = i + 1; } @@ -3924,7 +3900,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3946,7 +3922,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } else { if (pInfo->showRewrite == false) { @@ -4198,14 +4174,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; @@ -4282,7 +4252,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { count += 1; if (++pInfo->curPos >= size) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -4334,14 +4304,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -4712,7 +4675,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; if (tableListSize == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->tableStartIndex = 0; @@ -4731,7 +4694,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } else { stopGroupTableMergeScan(pOperator); if (pInfo->tableEndIndex >= tableListSize - 1) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; @@ -4852,13 +4815,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - pOperator->name = "TableMergeScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 700612e0ab..fc53623d44 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -53,11 +53,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); - pOperator->name = "SortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SortOperator", QUERY_NODE_PHYSICAL_PLAN_SORT, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; @@ -214,7 +210,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -428,7 +424,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); if (pInfo->prefetchedSortInput == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; @@ -453,7 +449,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { beginSortGroup(pOperator); } else if (pInfo->childOpStatus == CHILD_OP_FINISHED) { finishSortGroup(pOperator); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -509,14 +505,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort } pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); - - pOperator->name = "GroupSortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, getGroupSortExplainExecInfo); @@ -705,7 +694,7 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } return pBlock; @@ -774,13 +763,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->bufPageSize = getProperSortPageSize(rowSize); pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. - pOperator->name = "MultiwayMerge"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index a6ac28829c..7c9d73ad13 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1443,7 +1443,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRes, "stream fill"); return pInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1512,7 +1512,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } if (pInfo->pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1690,13 +1690,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi } pInfo->srcRowIndex = 0; - - pOperator->name = "StreamFillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6d8282ff5d..ca250f304a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1221,7 +1221,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -1232,7 +1232,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1269,7 +1269,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1739,7 +1739,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); - pOperator->pTaskInfo = pTaskInfo; pInfo->win = pTaskInfo->window; pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -1777,12 +1776,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); @@ -1890,7 +1884,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1933,7 +1927,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2281,7 +2275,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2330,7 +2324,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2342,7 +2336,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2365,7 +2359,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2386,7 +2380,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } else { @@ -2448,7 +2442,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2463,7 +2457,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2557,13 +2551,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pScanInfo->cond.twindows = pInfo->win; pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; - pOperator->name = "TimeSliceOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); @@ -2633,13 +2621,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); @@ -2711,12 +2694,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW goto _error; } - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - + setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); pOperator->pTaskInfo = pTaskInfo; @@ -3134,7 +3112,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer @@ -4024,7 +4002,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4124,7 +4102,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4191,13 +4169,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamSessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); @@ -4248,7 +4224,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -4321,7 +4297,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4332,20 +4308,21 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pOperator == NULL) { goto _error; } + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { - pInfo->isFinal = true; - pOperator->name = "StreamSessionFinalAggOperator"; - } else { - pInfo->isFinal = false; + pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); + char* name = (pInfo->isFinal)? "StreamSessionFinalAggOperator":"StreamSessionSemiAggOperator"; + + if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->name = "StreamSessionSemiAggOperator"; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); } + setOperatorInfo(pOperator, name, pPhyNode->type , false, OP_NOT_OPENED, pInfo, pTaskInfo); + pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); @@ -4575,7 +4552,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4641,7 +4618,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4706,12 +4683,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamStateAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, @@ -4861,7 +4833,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -4986,13 +4958,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&iaInfo->binfo.resultRowInfo); blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); - - pOperator->name = "TimeMergeAlignedIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = miaInfo; + setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); @@ -5239,7 +5205,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } size_t rows = pRes->info.rows; @@ -5298,14 +5264,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge } initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); - - pOperator->name = "TimeMergeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pMergeIntervalInfo; - + setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); @@ -5351,7 +5310,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval, &pInfo->delKey); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); return NULL; } @@ -5535,11 +5494,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamIntervalOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 55642fa1a1..079e553b07 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3112,6 +3112,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } #else if (!pInputCol->hasNull) { + numOfElems = 1; + int32_t round = pInput->numOfRows >> 2; int32_t reminder = pInput->numOfRows & 0x03; @@ -3143,7 +3145,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { - numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { char* data = colDataGetData(pInputCol, i); doSaveCurrentVal(pCtx, i, pts[i], type, data); @@ -3173,7 +3174,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); } - SET_VAL(pResInfo, numOfElems, 1); + +// SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } From 808ee3775d9edb086d30f931cb7f3328fb3d1bf4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 19:42:30 +0800 Subject: [PATCH 31/56] fix: wait until db created while process use db req --- source/dnode/mnode/impl/src/mndDb.c | 10 +++++++++- source/dnode/mnode/impl/src/mndTrans.c | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f2a4462bf5..8de1e976fc 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -606,7 +606,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } else { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and response after trans finished", createReq.db); + mInfo("db:%s, is creating and createdb response after trans finished", createReq.db); code = TSDB_CODE_ACTION_IN_PROGRESS; goto _OVER; } else { @@ -1225,6 +1225,14 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { usedbRsp.vgVersion = usedbReq.vgVersion; usedbRsp.errCode = terrno; + if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { + mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } + } + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ac05598bdc..db878d72b9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -938,11 +938,15 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { for (int32_t i = 0; i < size; ++i) { SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); if (pInfo->handle != NULL) { - mInfo("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), - pInfo->ahandle); if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } + if (i != 0 && code == 0) { + code = TSDB_CODE_RPC_REDIRECT; + } + mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), + pInfo->ahandle); + SRpcMsg rspMsg = {.code = code, .info = *pInfo}; if (pTrans->originRpcType == TDMT_MND_CREATE_DB) { From b595c07149f298c3fcbc8b9b266582d209aad401 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 19:47:22 +0800 Subject: [PATCH 32/56] fix: wait until db created while process use db req --- source/dnode/mnode/impl/src/mndDb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8de1e976fc..cdd0ce9fc5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1263,7 +1263,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { pReq->info.rspLen = contLen; _OVER: - if (code != 0) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } From e5bd00f0ae4feba93b87ff4b0f4e44ebc492cb21 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 9 Nov 2022 20:29:32 +0800 Subject: [PATCH 33/56] ci:add the env that builds with sanitizer --- Jenkinsfile2 | 4 +- tests/develop-test/fulltest.sh | 21 - tests/parallel_test/cases.task | 1000 ++++++++++++++++++++++++ tests/parallel_test/container_build.sh | 14 +- tests/parallel_test/run.sh | 7 +- tests/parallel_test/run_container.sh | 25 +- tests/script/jenkins/basic.txt | 457 ----------- tests/system-test/fulltest.sh | 656 ---------------- 8 files changed, 1038 insertions(+), 1146 deletions(-) delete mode 100644 tests/develop-test/fulltest.sh create mode 100644 tests/parallel_test/cases.task delete mode 100644 tests/script/jenkins/basic.txt delete mode 100755 tests/system-test/fulltest.sh diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 9e89e5a2f8..33c3ef55c9 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -430,8 +430,6 @@ pipeline { rm -rf ${WKC}/debug cd ${WKC}/tests/parallel_test time ./container_build.sh -w ${WKDIR} -t 10 -e - rm -f /tmp/cases.task - ./collect_cases.sh -e ''' def extra_param = "" def log_server_file = "/home/log_server.json" @@ -462,7 +460,7 @@ pipeline { cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 date - ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' + ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' ''' } } diff --git a/tests/develop-test/fulltest.sh b/tests/develop-test/fulltest.sh deleted file mode 100644 index e986ed6966..0000000000 --- a/tests/develop-test/fulltest.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e -set -x - -python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/commandline.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/limit_offset_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_interlace.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_telnet_alltypes.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/taosadapter_json.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task new file mode 100644 index 0000000000..3d3eca4e6d --- /dev/null +++ b/tests/parallel_test/cases.task @@ -0,0 +1,1000 @@ +,,y,unit-test,bash test.sh +,,,script,./test.sh -f tsim/user/basic.sim +,,,script,./test.sh -f tsim/user/password.sim +,,,script,./test.sh -f tsim/user/privilege_db.sim +,,,script,./test.sh -f tsim/user/privilege_sysinfo.sim +,,,script,./test.sh -f tsim/db/alter_option.sim +,,,script,./test.sh -f tsim/db/alter_replica_13.sim +,,,script,./test.sh -f tsim/db/alter_replica_31.sim +,,,script,./test.sh -f tsim/db/basic1.sim +,,,script,./test.sh -f tsim/db/basic2.sim +,,,script,./test.sh -f tsim/db/basic3.sim +,,,script,./test.sh -f tsim/db/basic4.sim +,,,script,./test.sh -f tsim/db/basic5.sim +,,,script,./test.sh -f tsim/db/basic6.sim +,,,script,./test.sh -f tsim/db/commit.sim +,,,script,./test.sh -f tsim/db/create_all_options.sim +,,,script,./test.sh -f tsim/db/delete_reuse1.sim +,,,script,./test.sh -f tsim/db/delete_reuse2.sim +,,,script,./test.sh -f tsim/db/delete_reusevnode.sim +,,,script,./test.sh -f tsim/db/delete_reusevnode2.sim +,,,script,./test.sh -f tsim/db/delete_writing1.sim +,,,script,./test.sh -f tsim/db/delete_writing2.sim +,,,script,./test.sh -f tsim/db/error1.sim +,,,script,./test.sh -f tsim/db/keep.sim +,,,script,./test.sh -f tsim/db/len.sim +,,,script,./test.sh -f tsim/db/repeat.sim +,,,script,./test.sh -f tsim/db/show_create_db.sim +,,,script,./test.sh -f tsim/db/show_create_table.sim +,,,script,./test.sh -f tsim/db/tables.sim +,,,script,./test.sh -f tsim/db/taosdlog.sim +,,,script,./test.sh -f tsim/dnode/balance_replica1.sim +,,,script,./test.sh -f tsim/dnode/balance_replica3.sim +,,,script,./test.sh -f tsim/dnode/balance1.sim +,,,script,./test.sh -f tsim/dnode/balance2.sim +,,,script,./test.sh -f tsim/dnode/balance3.sim +,,,script,./test.sh -f tsim/dnode/balancex.sim +,,,script,./test.sh -f tsim/dnode/create_dnode.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_force.sim +,,,script,./test.sh -f tsim/dnode/offline_reason.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +,,,script,./test.sh -f tsim/dnode/vnode_clean.sim +,,,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim +,,,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim +,,,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim +,,,script,./test.sh -f tsim/import/basic.sim +,,,script,./test.sh -f tsim/import/commit.sim +,,,script,./test.sh -f tsim/import/large.sim +,,,script,./test.sh -f tsim/import/replica1.sim +,,,script,./test.sh -f tsim/insert/backquote.sim +,,,script,./test.sh -f tsim/insert/basic.sim +,,,script,./test.sh -f tsim/insert/basic0.sim +,,,script,./test.sh -f tsim/insert/basic1.sim +,,,script,./test.sh -f tsim/insert/basic2.sim +,,,script,./test.sh -f tsim/insert/commit-merge0.sim +,,,script,./test.sh -f tsim/insert/insert_drop.sim +,,,script,./test.sh -f tsim/insert/insert_select.sim +,,,script,./test.sh -f tsim/insert/null.sim +,,,script,./test.sh -f tsim/insert/query_block1_file.sim +,,,script,./test.sh -f tsim/insert/query_block1_memory.sim +,,,script,./test.sh -f tsim/insert/query_block2_file.sim +,,,script,./test.sh -f tsim/insert/query_block2_memory.sim +,,,script,./test.sh -f tsim/insert/query_file_memory.sim +,,,script,./test.sh -f tsim/insert/query_multi_file.sim +,,,script,./test.sh -f tsim/insert/tcp.sim +,,,script,./test.sh -f tsim/insert/update0.sim +,,,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,,script,./test.sh -f tsim/parser/alter__for_community_version.sim +,,,script,./test.sh -f tsim/parser/alter_column.sim +,,,script,./test.sh -f tsim/parser/alter_stable.sim +,,,script,./test.sh -f tsim/parser/alter.sim +,,,script,./test.sh -f tsim/parser/alter1.sim +,,,script,./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim +,,,script,./test.sh -f tsim/parser/auto_create_tb.sim +,,,script,./test.sh -f tsim/parser/between_and.sim +,,,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim +,,,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim +,,,script,./test.sh -f tsim/parser/columnValue_bigint.sim +,,,script,./test.sh -f tsim/parser/columnValue_bool.sim +,,,script,./test.sh -f tsim/parser/columnValue_double.sim +,,,script,./test.sh -f tsim/parser/columnValue_float.sim +,,,script,./test.sh -f tsim/parser/columnValue_int.sim +,,,script,./test.sh -f tsim/parser/columnValue_smallint.sim +,,,script,./test.sh -f tsim/parser/columnValue_tinyint.sim +,,,script,./test.sh -f tsim/parser/columnValue_unsign.sim +,,,script,./test.sh -f tsim/parser/commit.sim +,,,script,./test.sh -f tsim/parser/condition.sim +,,,script,./test.sh -f tsim/parser/constCol.sim +,,,script,./test.sh -f tsim/parser/create_db.sim +,,,script,./test.sh -f tsim/parser/create_mt.sim +,,,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim +,,,script,./test.sh -f tsim/parser/create_tb.sim +,,,script,./test.sh -f tsim/parser/dbtbnameValidate.sim +,,,script,./test.sh -f tsim/parser/distinct.sim +,,,script,./test.sh -f tsim/parser/fill_us.sim +,,,script,./test.sh -f tsim/parser/fill.sim +,,,script,./test.sh -f tsim/parser/first_last.sim +,,,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim +,,,script,./test.sh -f tsim/parser/function.sim +,,,script,./test.sh -f tsim/parser/groupby-basic.sim +,,,script,./test.sh -f tsim/parser/groupby.sim +,,,script,./test.sh -f tsim/parser/having_child.sim +,,,script,./test.sh -f tsim/parser/having.sim +,,,script,./test.sh -f tsim/parser/import_commit1.sim +,,,script,./test.sh -f tsim/parser/import_commit2.sim +,,,script,./test.sh -f tsim/parser/import_commit3.sim +,,,script,./test.sh -f tsim/parser/import_file.sim +,,,script,./test.sh -f tsim/parser/import.sim +,,,script,./test.sh -f tsim/parser/insert_multiTbl.sim +,,,script,./test.sh -f tsim/parser/insert_tb.sim +,,,script,./test.sh -f tsim/parser/join_manyblocks.sim +,,,script,./test.sh -f tsim/parser/join_multitables.sim +,,,script,./test.sh -f tsim/parser/join_multivnode.sim +,,,script,./test.sh -f tsim/parser/join.sim +,,,script,./test.sh -f tsim/parser/last_cache.sim +,,,script,./test.sh -f tsim/parser/last_groupby.sim +,,,script,./test.sh -f tsim/parser/lastrow.sim +,,,script,./test.sh -f tsim/parser/lastrow2.sim +,,,script,./test.sh -f tsim/parser/like.sim +,,,script,./test.sh -f tsim/parser/limit.sim +,,,script,./test.sh -f tsim/parser/limit1.sim +,,,script,./test.sh -f tsim/parser/mixed_blocks.sim +,,,script,./test.sh -f tsim/parser/nchar.sim +,,,script,./test.sh -f tsim/parser/nestquery.sim +,,,script,./test.sh -f tsim/parser/null_char.sim +,,,script,./test.sh -f tsim/parser/precision_ns.sim +,,,script,./test.sh -f tsim/parser/projection_limit_offset.sim +,,,script,./test.sh -f tsim/parser/regex.sim +,,,script,./test.sh -f tsim/parser/select_across_vnodes.sim +,,,script,./test.sh -f tsim/parser/select_distinct_tag.sim +,,,script,./test.sh -f tsim/parser/select_from_cache_disk.sim +,,,script,./test.sh -f tsim/parser/select_with_tags.sim +,,,script,./test.sh -f tsim/parser/selectResNum.sim +,,,script,./test.sh -f tsim/parser/set_tag_vals.sim +,,,script,./test.sh -f tsim/parser/single_row_in_tb.sim +,,,script,./test.sh -f tsim/parser/sliding.sim +,,,script,./test.sh -f tsim/parser/slimit_alter_tags.sim +,,,script,./test.sh -f tsim/parser/slimit.sim +,,,script,./test.sh -f tsim/parser/slimit1.sim +,,,script,./test.sh -f tsim/parser/stableOp.sim +,,,script,./test.sh -f tsim/parser/tags_dynamically_specifiy.sim +,,,script,./test.sh -f tsim/parser/tags_filter.sim +,,,script,./test.sh -f tsim/parser/tbnameIn.sim +,,,script,./test.sh -f tsim/parser/timestamp.sim +,,,script,./test.sh -f tsim/parser/top_groupby.sim +,,,script,./test.sh -f tsim/parser/topbot.sim +,,,script,./test.sh -f tsim/parser/union.sim +,,,script,./test.sh -f tsim/parser/union_sysinfo.sim +,,,script,./test.sh -f tsim/parser/where.sim +,,,script,./test.sh -f tsim/query/charScalarFunction.sim +,,,script,./test.sh -f tsim/query/explain.sim +,,,script,./test.sh -f tsim/query/interval-offset.sim +,,,script,./test.sh -f tsim/query/interval.sim +,,,script,./test.sh -f tsim/query/scalarFunction.sim +,,,script,./test.sh -f tsim/query/scalarNull.sim +,,,script,./test.sh -f tsim/query/session.sim +,,,script,./test.sh -f tsim/query/udf.sim +,,,script,./test.sh -f tsim/qnode/basic1.sim +,,,script,./test.sh -f tsim/snode/basic1.sim +,,,script,./test.sh -f tsim/mnode/basic1.sim +,,,script,./test.sh -f tsim/mnode/basic2.sim +,,,script,./test.sh -f tsim/mnode/basic3.sim +,,,script,./test.sh -f tsim/mnode/basic4.sim +,,,script,./test.sh -f tsim/mnode/basic5.sim +,,,script,./test.sh -f tsim/show/basic.sim +,,,script,./test.sh -f tsim/table/autocreate.sim +,,,script,./test.sh -f tsim/table/basic1.sim +,,,script,./test.sh -f tsim/table/basic2.sim +,,,script,./test.sh -f tsim/table/basic3.sim +,,,script,./test.sh -f tsim/table/bigint.sim +,,,script,./test.sh -f tsim/table/binary.sim +,,,script,./test.sh -f tsim/table/bool.sim +,,,script,./test.sh -f tsim/table/column_name.sim +,,,script,./test.sh -f tsim/table/column_num.sim +,,,script,./test.sh -f tsim/table/column_value.sim +,,,script,./test.sh -f tsim/table/column2.sim +,,,script,./test.sh -f tsim/table/createmulti.sim +,,,script,./test.sh -f tsim/table/date.sim +,,,script,./test.sh -f tsim/table/db.table.sim +,,,script,./test.sh -f tsim/table/delete_reuse1.sim +,,,script,./test.sh -f tsim/table/delete_reuse2.sim +,,,script,./test.sh -f tsim/table/delete_writing.sim +,,,script,./test.sh -f tsim/table/describe.sim +,,,script,./test.sh -f tsim/table/double.sim +,,,script,./test.sh -f tsim/table/float.sim +,,,script,./test.sh -f tsim/table/hash.sim +,,,script,./test.sh -f tsim/table/int.sim +,,,script,./test.sh -f tsim/table/limit.sim +,,,script,./test.sh -f tsim/table/smallint.sim +,,,script,./test.sh -f tsim/table/table_len.sim +,,,script,./test.sh -f tsim/table/table.sim +,,,script,./test.sh -f tsim/table/tinyint.sim +,,,script,./test.sh -f tsim/table/vgroup.sim +,,,script,./test.sh -f tsim/stream/basic0.sim -g +,,,script,./test.sh -f tsim/stream/basic2.sim +,,,script,./test.sh -f tsim/stream/drop_stream.sim +,,,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim +,,,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim +,,,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,,script,./test.sh -f tsim/stream/distributeInterval0.sim +,,,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,,script,./test.sh -f tsim/stream/distributeSession0.sim +,,,script,./test.sh -f tsim/stream/session0.sim +,,,script,./test.sh -f tsim/stream/session1.sim +,,,script,./test.sh -f tsim/stream/state0.sim +,,,script,./test.sh -f tsim/stream/triggerInterval0.sim +,,,script,./test.sh -f tsim/stream/triggerSession0.sim +,,,script,./test.sh -f tsim/stream/partitionby.sim +,,,script,./test.sh -f tsim/stream/partitionby1.sim +,,,script,./test.sh -f tsim/stream/schedSnode.sim +,,,script,./test.sh -f tsim/stream/windowClose.sim +,,,script,./test.sh -f tsim/stream/ignoreExpiredData.sim +,,,script,./test.sh -f tsim/stream/sliding.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnState.sim +,,,script,./test.sh -f tsim/stream/deleteInterval.sim +,,,script,./test.sh -f tsim/stream/deleteSession.sim +,,,script,./test.sh -f tsim/stream/deleteState.sim +,,,script,./test.sh -f tsim/stream/fillIntervalDelete0.sim +,,,script,./test.sh -f tsim/stream/fillIntervalDelete1.sim +,,,script,./test.sh -f tsim/stream/fillIntervalLinear.sim +,,,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +,,,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim +,,,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,,script,./test.sh -f tsim/trans/lossdata1.sim +,,,script,./test.sh -f tsim/trans/create_db.sim +,,,script,./test.sh -f tsim/tmq/basic1.sim +,,,script,./test.sh -f tsim/tmq/basic2.sim +,,,script,./test.sh -f tsim/tmq/basic3.sim +,,,script,./test.sh -f tsim/tmq/basic4.sim +,,,script,./test.sh -f tsim/tmq/basic1Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic2Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic3Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic4Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +,,,script,./test.sh -f tsim/tmq/topic.sim +,,,script,./test.sh -f tsim/tmq/snapshot.sim +,,,script,./test.sh -f tsim/tmq/snapshot1.sim +,,,script,./test.sh -f tsim/stable/alter_comment.sim +,,,script,./test.sh -f tsim/stable/alter_count.sim +,,,script,./test.sh -f tsim/stable/alter_import.sim +,,,script,./test.sh -f tsim/stable/alter_insert1.sim +,,,script,./test.sh -f tsim/stable/alter_insert2.sim +,,,script,./test.sh -f tsim/stable/alter_metrics.sim +,,,script,./test.sh -f tsim/stable/column_add.sim +,,,script,./test.sh -f tsim/stable/column_drop.sim +,,,script,./test.sh -f tsim/stable/column_modify.sim +,,,script,./test.sh -f tsim/stable/disk.sim +,,,script,./test.sh -f tsim/stable/dnode3.sim +,,,script,./test.sh -f tsim/stable/metrics.sim +,,,script,./test.sh -f tsim/stable/refcount.sim +,,,script,./test.sh -f tsim/stable/tag_add.sim +,,,script,./test.sh -f tsim/stable/tag_drop.sim +,,,script,./test.sh -f tsim/stable/tag_filter.sim +,,,script,./test.sh -f tsim/stable/tag_modify.sim +,,,script,./test.sh -f tsim/stable/tag_rename.sim +,,,script,./test.sh -f tsim/stable/values.sim +,,,script,./test.sh -f tsim/stable/vnode3.sim +,,,script,./test.sh -f tsim/stable/metrics_idx.sim +,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +,,,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +,,,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +,,,script,./test.sh -f tsim/valgrind/checkError1.sim +,,,script,./test.sh -f tsim/valgrind/checkError2.sim +,,,script,./test.sh -f tsim/valgrind/checkError3.sim +,,,script,./test.sh -f tsim/valgrind/checkError4.sim +,,,script,./test.sh -f tsim/valgrind/checkError5.sim +,,,script,./test.sh -f tsim/valgrind/checkError6.sim +,,,script,./test.sh -f tsim/valgrind/checkError7.sim +,,,script,./test.sh -f tsim/valgrind/checkError8.sim +,,,script,./test.sh -f tsim/valgrind/checkUdf.sim +,,,script,./test.sh -f tsim/vnode/replica3_basic.sim +,,,script,./test.sh -f tsim/vnode/replica3_repeat.sim +,,,script,./test.sh -f tsim/vnode/replica3_vgroup.sim +,,,script,./test.sh -f tsim/vnode/replica3_many.sim +,,,script,./test.sh -f tsim/vnode/replica3_import.sim +,,,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim +,,,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim +,,,script,./test.sh -f tsim/vnode/stable_dnode2.sim +,,,script,./test.sh -f tsim/vnode/stable_dnode3.sim +,,,script,./test.sh -f tsim/vnode/stable_replica3_dnode6.sim +,,,script,./test.sh -f tsim/vnode/stable_replica3_vnode3.sim +,,,script,./test.sh -f tsim/sync/3Replica1VgElect.sim +,,,script,./test.sh -f tsim/sync/3Replica5VgElect.sim +,,,script,./test.sh -f tsim/sync/oneReplica1VgElect.sim +,,,script,./test.sh -f tsim/sync/oneReplica5VgElect.sim +,,,script,./test.sh -f tsim/catalog/alterInCurrent.sim +,,,script,./test.sh -f tsim/scalar/in.sim +,,,script,./test.sh -f tsim/scalar/scalar.sim +,,,script,./test.sh -f tsim/scalar/filter.sim +,,,script,./test.sh -f tsim/scalar/caseWhen.sim +,,,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim +,,,script,./test.sh -f tsim/alter/dnode.sim +,,,script,./test.sh -f tsim/alter/table.sim +,,,script,./test.sh -f tsim/cache/new_metrics.sim +,,,script,./test.sh -f tsim/cache/restart_table.sim +,,,script,./test.sh -f tsim/cache/restart_metrics.sim +,,,script,./test.sh -f tsim/column/commit.sim +,,,script,./test.sh -f tsim/column/metrics.sim +,,,script,./test.sh -f tsim/column/table.sim +,,,script,./test.sh -f tsim/compress/commitlog.sim +,,,script,./test.sh -f tsim/compress/compress2.sim +,,,script,./test.sh -f tsim/compress/compress.sim +,,,script,./test.sh -f tsim/compress/uncompress.sim +,,,script,./test.sh -f tsim/compute/avg.sim +,,,script,./test.sh -f tsim/compute/block_dist.sim +,,,script,./test.sh -f tsim/compute/bottom.sim +,,,script,./test.sh -f tsim/compute/count.sim +,,,script,./test.sh -f tsim/compute/diff.sim +,,,script,./test.sh -f tsim/compute/diff2.sim +,,,script,./test.sh -f tsim/compute/first.sim +,,,script,./test.sh -f tsim/compute/interval.sim +,,,script,./test.sh -f tsim/compute/last_row.sim +,,,script,./test.sh -f tsim/compute/last.sim +,,,script,./test.sh -f tsim/compute/leastsquare.sim +,,,script,./test.sh -f tsim/compute/max.sim +,,,script,./test.sh -f tsim/compute/min.sim +,,,script,./test.sh -f tsim/compute/null.sim +,,,script,./test.sh -f tsim/compute/percentile.sim +,,,script,./test.sh -f tsim/compute/stddev.sim +,,,script,./test.sh -f tsim/compute/sum.sim +,,,script,./test.sh -f tsim/compute/top.sim +,,,script,./test.sh -f tsim/field/2.sim +,,,script,./test.sh -f tsim/field/3.sim +,,,script,./test.sh -f tsim/field/4.sim +,,,script,./test.sh -f tsim/field/5.sim +,,,script,./test.sh -f tsim/field/6.sim +,,,script,./test.sh -f tsim/field/binary.sim +,,,script,./test.sh -f tsim/field/bigint.sim +,,,script,./test.sh -f tsim/field/bool.sim +,,,script,./test.sh -f tsim/field/double.sim +,,,script,./test.sh -f tsim/field/float.sim +,,,script,./test.sh -f tsim/field/int.sim +,,,script,./test.sh -f tsim/field/single.sim +,,,script,./test.sh -f tsim/field/smallint.sim +,,,script,./test.sh -f tsim/field/tinyint.sim +,,,script,./test.sh -f tsim/field/unsigined_bigint.sim +,,,script,./test.sh -f tsim/vector/metrics_field.sim +,,,script,./test.sh -f tsim/vector/metrics_mix.sim +,,,script,./test.sh -f tsim/vector/metrics_query.sim +,,,script,./test.sh -f tsim/vector/metrics_tag.sim +,,,script,./test.sh -f tsim/vector/metrics_time.sim +,,,script,./test.sh -f tsim/vector/multi.sim +,,,script,./test.sh -f tsim/vector/single.sim +,,,script,./test.sh -f tsim/vector/table_field.sim +,,,script,./test.sh -f tsim/vector/table_mix.sim +,,,script,./test.sh -f tsim/vector/table_query.sim +,,,script,./test.sh -f tsim/vector/table_time.sim +,,,script,./test.sh -f tsim/wal/kill.sim +,,,script,./test.sh -f tsim/tag/3.sim +,,,script,./test.sh -f tsim/tag/4.sim +,,,script,./test.sh -f tsim/tag/5.sim +,,,script,./test.sh -f tsim/tag/6.sim +,,,script,./test.sh -f tsim/tag/add.sim +,,,script,./test.sh -f tsim/tag/bigint.sim +,,,script,./test.sh -f tsim/tag/binary_binary.sim +,,,script,./test.sh -f tsim/tag/binary.sim +,,,script,./test.sh -f tsim/tag/bool_binary.sim +,,,script,./test.sh -f tsim/tag/bool_int.sim +,,,script,./test.sh -f tsim/tag/bool.sim +,,,script,./test.sh -f tsim/tag/change.sim +,,,script,./test.sh -f tsim/tag/column.sim +,,,script,./test.sh -f tsim/tag/commit.sim +,,,script,./test.sh -f tsim/tag/create.sim +,,,script,./test.sh -f tsim/tag/delete.sim +,,,script,./test.sh -f tsim/tag/double.sim +,,,script,./test.sh -f tsim/tag/filter.sim +,,,script,./test.sh -f tsim/tag/float.sim +,,,script,./test.sh -f tsim/tag/int_binary.sim +,,,script,./test.sh -f tsim/tag/int_float.sim +,,,script,./test.sh -f tsim/tag/int.sim +,,,script,./test.sh -f tsim/tag/set.sim +,,,script,./test.sh -f tsim/tag/smallint.sim +,,,script,./test.sh -f tsim/tag/tinyint.sim +,,,script,./test.sh -f tsim/tag/drop_tag.sim +,,,script,./test.sh -f tsim/tag/tbNameIn.sim +,,,script,./test.sh -f tmp/monitor.sim +,,,system-test,python3 ./test.py -f 0-others/taosShell.py +,,,system-test,python3 ./test.py -f 0-others/taosShellError.py +,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py +,,,system-test,python3 ./test.py -f 0-others/telemetry.py +,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py +,,,system-test,python3 ./test.py -f 0-others/udfTest.py +,,,system-test,python3 ./test.py -f 0-others/udf_create.py +,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py +,,,system-test,python3 ./test.py -f 0-others/cachemodel.py +,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,,system-test,python3 ./test.py -f 0-others/udf_cfg2.py +,,,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +,,,system-test,python3 ./test.py -f 0-others/sysinfo.py +,,,system-test,python3 ./test.py -f 0-others/user_control.py +,,,system-test,python3 ./test.py -f 0-others/fsync.py +,,,system-test,python3 ./test.py -f 0-others/compatibility.py +,,,system-test,python3 ./test.py -f 1-insert/alter_database.py +,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +,,,system-test,python3 ./test.py -f 1-insert/alter_stable.py +,,,system-test,python3 ./test.py -f 1-insert/alter_table.py +,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +,,,system-test,python3 ./test.py -f 1-insert/table_comment.py +,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py +,,,system-test,python3 ./test.py -f 1-insert/block_wise.py +,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py +,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py +,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py +,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py -R +,,,system-test,python3 ./test.py -f 1-insert/update_data_muti_rows.py +,,,system-test,python3 ./test.py -f 1-insert/db_tb_name_check.py +,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py +,,,system-test,python3 ./test.py -f 0-others/show.py +,,,system-test,python3 ./test.py -f 2-query/abs.py +,,,system-test,python3 ./test.py -f 2-query/abs.py -R +,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py +,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R +,,,system-test,python3 ./test.py -f 2-query/apercentile.py +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R +,,,system-test,python3 ./test.py -f 2-query/arccos.py +,,,system-test,python3 ./test.py -f 2-query/arccos.py -R +,,,system-test,python3 ./test.py -f 2-query/arcsin.py +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -R +,,,system-test,python3 ./test.py -f 2-query/arctan.py +,,,system-test,python3 ./test.py -f 2-query/arctan.py -R +,,,system-test,python3 ./test.py -f 2-query/avg.py +,,,system-test,python3 ./test.py -f 2-query/avg.py -R +,,,system-test,python3 ./test.py -f 2-query/between.py +,,,system-test,python3 ./test.py -f 2-query/between.py -R +,,,system-test,python3 ./test.py -f 2-query/bottom.py +,,,system-test,python3 ./test.py -f 2-query/bottom.py -R +,,,system-test,python3 ./test.py -f 2-query/cast.py +,,,system-test,python3 ./test.py -f 2-query/cast.py -R +,,,system-test,python3 ./test.py -f 2-query/ceil.py +,,,system-test,python3 ./test.py -f 2-query/ceil.py -R +,,,system-test,python3 ./test.py -f 2-query/char_length.py +,,,system-test,python3 ./test.py -f 2-query/char_length.py -R +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -R +,,,system-test,python3 ./test.py -f 2-query/concat.py +,,,system-test,python3 ./test.py -f 2-query/concat.py -R +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R +,,,system-test,python3 ./test.py -f 2-query/cos.py +,,,system-test,python3 ./test.py -f 2-query/cos.py -R +,,,system-test,python3 ./test.py -f 2-query/count_partition.py +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -R +,,,system-test,python3 ./test.py -f 2-query/count.py +,,,system-test,python3 ./test.py -f 2-query/count.py -R +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +,,,system-test,python3 ./test.py -f 2-query/db.py +,,,system-test,python3 ./test.py -f 2-query/db.py -R +,,,system-test,python3 ./test.py -f 2-query/diff.py +,,,system-test,python3 ./test.py -f 2-query/diff.py -R +,,,system-test,python3 ./test.py -f 2-query/distinct.py +,,,system-test,python3 ./test.py -f 2-query/distinct.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -R +,,,system-test,python3 ./test.py -f 2-query/explain.py +,,,system-test,python3 ./test.py -f 2-query/explain.py -R +,,,system-test,python3 ./test.py -f 2-query/first.py +,,,system-test,python3 ./test.py -f 2-query/first.py -R +,,,system-test,python3 ./test.py -f 2-query/floor.py +,,,system-test,python3 ./test.py -f 2-query/floor.py -R +,,,system-test,python3 ./test.py -f 2-query/function_null.py +,,,system-test,python3 ./test.py -f 2-query/function_null.py -R +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -R +,,,system-test,python3 ./test.py -f 2-query/histogram.py +,,,system-test,python3 ./test.py -f 2-query/histogram.py -R +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -R +,,,system-test,python3 ./test.py -f 2-query/interp.py +,,,system-test,python3 ./test.py -f 2-query/interp.py -R +,,,system-test,python3 ./test.py -f 2-query/irate.py +,,,system-test,python3 ./test.py -f 2-query/irate.py -R +,,,system-test,python3 ./test.py -f 2-query/join.py +,,,system-test,python3 ./test.py -f 2-query/join.py -R +,,,system-test,python3 ./test.py -f 2-query/last_row.py +,,,system-test,python3 ./test.py -f 2-query/last_row.py -R +,,,system-test,python3 ./test.py -f 2-query/last.py +,,,system-test,python3 ./test.py -f 2-query/last.py -R +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -R +,,,system-test,python3 ./test.py -f 2-query/length.py +,,,system-test,python3 ./test.py -f 2-query/length.py -R +,,,system-test,python3 ./test.py -f 2-query/log.py +,,,system-test,python3 ./test.py -f 2-query/log.py -R +,,,system-test,python3 ./test.py -f 2-query/lower.py +,,,system-test,python3 ./test.py -f 2-query/lower.py -R +,,,system-test,python3 ./test.py -f 2-query/ltrim.py +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R +,,,system-test,python3 ./test.py -f 2-query/mavg.py +,,,system-test,python3 ./test.py -f 2-query/mavg.py -R +,,,system-test,python3 ./test.py -f 2-query/max_partition.py +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R +,,,system-test,python3 ./test.py -f 2-query/max.py +,,,system-test,python3 ./test.py -f 2-query/max.py -R +,,,system-test,python3 ./test.py -f 2-query/min.py +,,,system-test,python3 ./test.py -f 2-query/min.py -R +,,,system-test,python3 ./test.py -f 2-query/mode.py +,,,system-test,python3 ./test.py -f 2-query/mode.py -R +,,,system-test,python3 ./test.py -f 2-query/Now.py +,,,system-test,python3 ./test.py -f 2-query/Now.py -R +,,,system-test,python3 ./test.py -f 2-query/percentile.py +,,,system-test,python3 ./test.py -f 2-query/percentile.py -R +,,,system-test,python3 ./test.py -f 2-query/pow.py +,,,system-test,python3 ./test.py -f 2-query/pow.py -R +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R +,,,system-test,python3 ./test.py -f 2-query/round.py +,,,system-test,python3 ./test.py -f 2-query/round.py -R +,,,system-test,python3 ./test.py -f 2-query/rtrim.py +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R +,,,system-test,python3 ./test.py -f 2-query/sample.py +,,,system-test,python3 ./test.py -f 2-query/sample.py -R +,,,system-test,python3 ./test.py -f 2-query/sin.py +,,,system-test,python3 ./test.py -f 2-query/sin.py -R +,,,system-test,python3 ./test.py -f 2-query/smaTest.py +,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R +,,,system-test,python3 ./test.py -f 2-query/sml.py +,,,system-test,python3 ./test.py -f 2-query/sml.py -R +,,,system-test,python3 ./test.py -f 2-query/spread.py +,,,system-test,python3 ./test.py -f 2-query/spread.py -R +,,,system-test,python3 ./test.py -f 2-query/sqrt.py +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -R +,,,system-test,python3 ./test.py -f 2-query/statecount.py +,,,system-test,python3 ./test.py -f 2-query/statecount.py -R +,,,system-test,python3 ./test.py -f 2-query/stateduration.py +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -R +,,,system-test,python3 ./test.py -f 2-query/substr.py +,,,system-test,python3 ./test.py -f 2-query/substr.py -R +,,,system-test,python3 ./test.py -f 2-query/sum.py +,,,system-test,python3 ./test.py -f 2-query/sum.py -R +,,,system-test,python3 ./test.py -f 2-query/tail.py +,,,system-test,python3 ./test.py -f 2-query/tail.py -R +,,,system-test,python3 ./test.py -f 2-query/tan.py +,,,system-test,python3 ./test.py -f 2-query/tan.py -R +,,,system-test,python3 ./test.py -f 2-query/Timediff.py +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -R +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -R +,,,system-test,python3 ./test.py -f 2-query/timezone.py +,,,system-test,python3 ./test.py -f 2-query/timezone.py -R +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -R +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -R +,,,system-test,python3 ./test.py -f 2-query/Today.py +,,,system-test,python3 ./test.py -f 2-query/Today.py -R +,,,system-test,python3 ./test.py -f 2-query/top.py +,,,system-test,python3 ./test.py -f 2-query/top.py -R +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R +,,,system-test,python3 ./test.py -f 2-query/twa.py +,,,system-test,python3 ./test.py -f 2-query/twa.py -R +,,,system-test,python3 ./test.py -f 2-query/union.py +,,,system-test,python3 ./test.py -f 2-query/union.py -R +,,,system-test,python3 ./test.py -f 2-query/unique.py +,,,system-test,python3 ./test.py -f 2-query/unique.py -R +,,,system-test,python3 ./test.py -f 2-query/upper.py +,,,system-test,python3 ./test.py -f 2-query/upper.py -R +,,,system-test,python3 ./test.py -f 2-query/varchar.py +,,,system-test,python3 ./test.py -f 2-query/varchar.py -R +,,,system-test,python3 ./test.py -f 1-insert/update_data.py +,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py +,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py +,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py +,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py +,,,system-test,python3 ./test.py -f 1-insert/keep_expired.py +,,,system-test,python3 ./test.py -f 2-query/join2.py +,,,system-test,python3 ./test.py -f 2-query/union1.py +,,,system-test,python3 ./test.py -f 2-query/concat2.py +,,,system-test,python3 ./test.py -f 2-query/json_tag.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py +,,,system-test,python3 ./test.py -f 2-query/stablity.py +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py +,,,system-test,python3 ./test.py -f 2-query/elapsed.py +,,,system-test,python3 ./test.py -f 2-query/csum.py +,,,system-test,python3 ./test.py -f 2-query/function_diff.py +,,,system-test,python3 ./test.py -f 2-query/queryQnode.py +,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py +,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 +,,,system-test,python3 ./test.py -f 7-tmq/create_wrong_topic.py +,,,system-test,python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +,,,system-test,python3 ./test.py -f 7-tmq/basic5.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb0.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb1.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb2.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb3.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb4.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb0.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb1.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb2.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb3.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb4.py +,,,system-test,python3 ./test.py -f 7-tmq/db.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqError.py +,,,system-test,python3 ./test.py -f 7-tmq/schema.py +,,,system-test,python3 ./test.py -f 7-tmq/stbFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqCheckData.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqCheckData1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqShow.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqAlterSchema.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropStb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUdf.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py +,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py +,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py +,,,system-test,python3 ./test.py -f 2-query/between.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/length.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/join.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/max.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/min.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/count.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/last.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/first.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/top.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/round.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/log.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/between.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/length.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/join.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/max.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/min.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/count.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/last.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/first.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/top.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/round.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/log.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/between.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/length.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/join.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/max.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/min.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/count.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/last.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/first.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/top.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/round.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/log.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R +,,,docs-examples-test,bash python.sh +,,,docs-examples-test,bash node.sh +,,,docs-examples-test,bash csharp.sh +,,,docs-examples-test,bash jdbc.sh +,,,docs-examples-test,bash go.sh diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 52864d668f..2257ba6829 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -44,16 +44,26 @@ ulimit -c unlimited if [ $ent -eq 0 ]; then REP_DIR=/home/TDengine - REP_MOUNT_PARAM=$WORKDIR/TDengine:/home/TDengine + REP_REAL_PATH=$WORKDIR/TDengine + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine else REP_DIR=/home/TDinternal - REP_MOUNT_PARAM=$WORKDIR/TDinternal:/home/TDinternal + REP_REAL_PATH=$WORKDIR/TDinternal + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal fi docker run \ -v $REP_MOUNT_PARAM \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT" +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan + +docker run \ + -v $REP_MOUNT_PARAM \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT" + +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan + ret=$? exit $ret diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 26f481e571..e7bab707ef 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -164,8 +164,9 @@ function run_thread() { if [ -z "$case_redo_time" ]; then case_redo_time=${DEFAULT_RETRY_TIME:-2} fi - local exec_dir=`echo "$line"|cut -d, -f3` - local case_cmd=`echo "$line"|cut -d, -f4` + local case_build_san=`echo "$line"|cut -d, -f3` + local exec_dir=`echo "$line"|cut -d, -f4` + local case_cmd=`echo "$line"|cut -d, -f5` local case_file="" echo "$case_cmd"|grep -q "\.sh" if [ $? -eq 0 ]; then @@ -191,7 +192,7 @@ function run_thread() { if [ ! -z "$case_path" ]; then mkdir -p $log_dir/$case_path fi - cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param}" + cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param} -s ${case_build_san}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index f58aaaf29d..9083510d10 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -8,11 +8,12 @@ function usage() { echo -e "\t -t thread number" echo -e "\t -e enterprise edition" echo -e "\t -o default timeout value" + echo -e "\t -s build with sanitizer" echo -e "\t -h help" } ent=0 -while getopts "w:d:c:t:o:eh" opt; do +while getopts "w:d:c:t:o:e:sh" opt; do case $opt in w) WORKDIR=$OPTARG @@ -32,6 +33,9 @@ while getopts "w:d:c:t:o:eh" opt; do o) extra_param="-o $OPTARG" ;; + s) + buildSan="-o $OPTARG" + ;; h) usage exit 0 @@ -60,23 +64,36 @@ if [ -z "$thread_no" ]; then usage exit 1 fi + +#select whether the compilation environment includes sanitizer +if [ "${buildSan}" == "y" ]; then + DEBUGPATH="buildSan" +elif [[ "${buildSan}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then + DEBUGPATH="debugNoSan" +else + usage + exit 1 +fi + if [ $ent -ne 0 ]; then # enterprise edition extra_param="$extra_param -e" INTERNAL_REPDIR=$WORKDIR/TDinternal REPDIR=$INTERNAL_REPDIR/community + REPDIR_DEBUG=$WORKDIR/DEBUGPATH/ CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" - REP_MOUNT_LIB="$INTERNAL_REPDIR/debug/build/lib:/home/TDinternal/debug/build/lib:ro" + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" else # community edition REPDIR=$WORKDIR/TDengine + REPDIR_DEBUG=$WORKDIR/DEBUGPATH/ CONTAINER_TESTDIR=/home/TDengine SIM_DIR=/home/TDengine/sim REP_MOUNT_PARAM="$REPDIR:/home/TDengine" - REP_MOUNT_LIB="$REPDIR/debug/build/lib:/home/TDengine/debug/build/lib:ro" + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDengine/debug/:ro" fi @@ -107,7 +124,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ - -v $REP_MOUNT_LIB \ + -v $REP_MOUNT_DEBUG \ -v $MOUNT_DIR \ -v ${SOURCEDIR}:/usr/local/src/ \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt deleted file mode 100644 index e6fdecf35e..0000000000 --- a/tests/script/jenkins/basic.txt +++ /dev/null @@ -1,457 +0,0 @@ - -#======================b1-start=============== - -# ---- user ---- -./test.sh -f tsim/user/basic.sim -./test.sh -f tsim/user/password.sim -./test.sh -f tsim/user/privilege_db.sim -./test.sh -f tsim/user/privilege_sysinfo.sim - -# ---- db ---- -./test.sh -f tsim/db/alter_option.sim -./test.sh -f tsim/db/alter_replica_13.sim -./test.sh -f tsim/db/alter_replica_31.sim -./test.sh -f tsim/db/basic1.sim -./test.sh -f tsim/db/basic2.sim -./test.sh -f tsim/db/basic3.sim -./test.sh -f tsim/db/basic4.sim -./test.sh -f tsim/db/basic5.sim -./test.sh -f tsim/db/basic6.sim -./test.sh -f tsim/db/commit.sim -./test.sh -f tsim/db/create_all_options.sim -./test.sh -f tsim/db/delete_reuse1.sim -./test.sh -f tsim/db/delete_reuse2.sim -./test.sh -f tsim/db/delete_reusevnode.sim -./test.sh -f tsim/db/delete_reusevnode2.sim -./test.sh -f tsim/db/delete_writing1.sim -./test.sh -f tsim/db/delete_writing2.sim -./test.sh -f tsim/db/error1.sim -./test.sh -f tsim/db/keep.sim -./test.sh -f tsim/db/len.sim -./test.sh -f tsim/db/repeat.sim -./test.sh -f tsim/db/show_create_db.sim -./test.sh -f tsim/db/show_create_table.sim -./test.sh -f tsim/db/tables.sim -./test.sh -f tsim/db/taosdlog.sim - -# ---- dnode -./test.sh -f tsim/dnode/balance_replica1.sim -./test.sh -f tsim/dnode/balance_replica3.sim -./test.sh -f tsim/dnode/balance1.sim -./test.sh -f tsim/dnode/balance2.sim -./test.sh -f tsim/dnode/balance3.sim -./test.sh -f tsim/dnode/balancex.sim -./test.sh -f tsim/dnode/create_dnode.sim -./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim -./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim -./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -./test.sh -f tsim/dnode/drop_dnode_force.sim -./test.sh -f tsim/dnode/offline_reason.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim -./test.sh -f tsim/dnode/vnode_clean.sim -./test.sh -f tsim/dnode/use_dropped_dnode.sim -./test.sh -f tsim/dnode/split_vgroup_replica1.sim -./test.sh -f tsim/dnode/split_vgroup_replica3.sim - -# ---- import ---- -./test.sh -f tsim/import/basic.sim -./test.sh -f tsim/import/commit.sim -./test.sh -f tsim/import/large.sim -./test.sh -f tsim/import/replica1.sim - -# ---- insert ---- -./test.sh -f tsim/insert/backquote.sim -./test.sh -f tsim/insert/basic.sim -./test.sh -f tsim/insert/basic0.sim -./test.sh -f tsim/insert/basic1.sim -./test.sh -f tsim/insert/basic2.sim -./test.sh -f tsim/insert/commit-merge0.sim -./test.sh -f tsim/insert/insert_drop.sim -./test.sh -f tsim/insert/insert_select.sim -./test.sh -f tsim/insert/null.sim -./test.sh -f tsim/insert/query_block1_file.sim -./test.sh -f tsim/insert/query_block1_memory.sim -./test.sh -f tsim/insert/query_block2_file.sim -./test.sh -f tsim/insert/query_block2_memory.sim -./test.sh -f tsim/insert/query_file_memory.sim -./test.sh -f tsim/insert/query_multi_file.sim -./test.sh -f tsim/insert/tcp.sim -./test.sh -f tsim/insert/update0.sim -./test.sh -f tsim/insert/update1_sort_merge.sim - -# ---- parser ---- -./test.sh -f tsim/parser/alter__for_community_version.sim -./test.sh -f tsim/parser/alter_column.sim -./test.sh -f tsim/parser/alter_stable.sim -./test.sh -f tsim/parser/alter.sim -./test.sh -f tsim/parser/alter1.sim -./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim -./test.sh -f tsim/parser/auto_create_tb.sim -./test.sh -f tsim/parser/between_and.sim -./test.sh -f tsim/parser/binary_escapeCharacter.sim -./test.sh -f tsim/parser/col_arithmetic_operation.sim -./test.sh -f tsim/parser/columnValue_bigint.sim -./test.sh -f tsim/parser/columnValue_bool.sim -./test.sh -f tsim/parser/columnValue_double.sim -./test.sh -f tsim/parser/columnValue_float.sim -./test.sh -f tsim/parser/columnValue_int.sim -./test.sh -f tsim/parser/columnValue_smallint.sim -./test.sh -f tsim/parser/columnValue_tinyint.sim -./test.sh -f tsim/parser/columnValue_unsign.sim -./test.sh -f tsim/parser/commit.sim -./test.sh -f tsim/parser/condition.sim -./test.sh -f tsim/parser/constCol.sim -./test.sh -f tsim/parser/create_db.sim -./test.sh -f tsim/parser/create_mt.sim -./test.sh -f tsim/parser/create_tb_with_tag_name.sim -./test.sh -f tsim/parser/create_tb.sim -./test.sh -f tsim/parser/dbtbnameValidate.sim -./test.sh -f tsim/parser/distinct.sim -# TD-17623 ./test.sh -f tsim/parser/fill_stb.sim -./test.sh -f tsim/parser/fill_us.sim -./test.sh -f tsim/parser/fill.sim -./test.sh -f tsim/parser/first_last.sim -./test.sh -f tsim/parser/fourArithmetic-basic.sim -./test.sh -f tsim/parser/function.sim -./test.sh -f tsim/parser/groupby-basic.sim -./test.sh -f tsim/parser/groupby.sim -./test.sh -f tsim/parser/having_child.sim -./test.sh -f tsim/parser/having.sim -./test.sh -f tsim/parser/import_commit1.sim -./test.sh -f tsim/parser/import_commit2.sim -./test.sh -f tsim/parser/import_commit3.sim -./test.sh -f tsim/parser/import_file.sim -./test.sh -f tsim/parser/import.sim -./test.sh -f tsim/parser/insert_multiTbl.sim -./test.sh -f tsim/parser/insert_tb.sim -# TD-18293 ./test.sh -f tsim/parser/interp.sim -./test.sh -f tsim/parser/join_manyblocks.sim -./test.sh -f tsim/parser/join_multitables.sim -./test.sh -f tsim/parser/join_multivnode.sim -./test.sh -f tsim/parser/join.sim -./test.sh -f tsim/parser/last_cache.sim -./test.sh -f tsim/parser/last_groupby.sim -./test.sh -f tsim/parser/lastrow.sim -./test.sh -f tsim/parser/lastrow2.sim -./test.sh -f tsim/parser/like.sim -./test.sh -f tsim/parser/limit.sim -./test.sh -f tsim/parser/limit1.sim -# TD-17623 ./test.sh -f tsim/parser/limit2.sim -./test.sh -f tsim/parser/mixed_blocks.sim -./test.sh -f tsim/parser/nchar.sim -./test.sh -f tsim/parser/nestquery.sim -./test.sh -f tsim/parser/null_char.sim -./test.sh -f tsim/parser/precision_ns.sim -./test.sh -f tsim/parser/projection_limit_offset.sim -./test.sh -f tsim/parser/regex.sim -./test.sh -f tsim/parser/select_across_vnodes.sim -./test.sh -f tsim/parser/select_distinct_tag.sim -./test.sh -f tsim/parser/select_from_cache_disk.sim -./test.sh -f tsim/parser/select_with_tags.sim -./test.sh -f tsim/parser/selectResNum.sim -./test.sh -f tsim/parser/set_tag_vals.sim -./test.sh -f tsim/parser/single_row_in_tb.sim -./test.sh -f tsim/parser/sliding.sim -./test.sh -f tsim/parser/slimit_alter_tags.sim -./test.sh -f tsim/parser/slimit.sim -./test.sh -f tsim/parser/slimit1.sim -./test.sh -f tsim/parser/stableOp.sim -./test.sh -f tsim/parser/tags_dynamically_specifiy.sim -./test.sh -f tsim/parser/tags_filter.sim -./test.sh -f tsim/parser/tbnameIn.sim -./test.sh -f tsim/parser/timestamp.sim -./test.sh -f tsim/parser/top_groupby.sim -./test.sh -f tsim/parser/topbot.sim -./test.sh -f tsim/parser/union.sim -./test.sh -f tsim/parser/union_sysinfo.sim -./test.sh -f tsim/parser/where.sim - -# ---- query ---- -./test.sh -f tsim/query/charScalarFunction.sim -./test.sh -f tsim/query/explain.sim -./test.sh -f tsim/query/interval-offset.sim -./test.sh -f tsim/query/interval.sim -./test.sh -f tsim/query/scalarFunction.sim -./test.sh -f tsim/query/scalarNull.sim -./test.sh -f tsim/query/session.sim -./test.sh -f tsim/query/udf.sim - -# ---- qnode -./test.sh -f tsim/qnode/basic1.sim - -# ---- snode ---- -./test.sh -f tsim/snode/basic1.sim - -# ---- mnode -./test.sh -f tsim/mnode/basic1.sim -./test.sh -f tsim/mnode/basic2.sim -./test.sh -f tsim/mnode/basic3.sim -./test.sh -f tsim/mnode/basic4.sim -./test.sh -f tsim/mnode/basic5.sim - -# ---- show ---- -./test.sh -f tsim/show/basic.sim - -# ---- table ---- -./test.sh -f tsim/table/autocreate.sim -./test.sh -f tsim/table/basic1.sim -./test.sh -f tsim/table/basic2.sim -./test.sh -f tsim/table/basic3.sim -./test.sh -f tsim/table/bigint.sim -./test.sh -f tsim/table/binary.sim -./test.sh -f tsim/table/bool.sim -./test.sh -f tsim/table/column_name.sim -./test.sh -f tsim/table/column_num.sim -./test.sh -f tsim/table/column_value.sim -./test.sh -f tsim/table/column2.sim -./test.sh -f tsim/table/createmulti.sim -./test.sh -f tsim/table/date.sim -./test.sh -f tsim/table/db.table.sim -./test.sh -f tsim/table/delete_reuse1.sim -./test.sh -f tsim/table/delete_reuse2.sim -./test.sh -f tsim/table/delete_writing.sim -./test.sh -f tsim/table/describe.sim -./test.sh -f tsim/table/double.sim -./test.sh -f tsim/table/float.sim -./test.sh -f tsim/table/hash.sim -./test.sh -f tsim/table/int.sim -./test.sh -f tsim/table/limit.sim -./test.sh -f tsim/table/smallint.sim -./test.sh -f tsim/table/table_len.sim -./test.sh -f tsim/table/table.sim -./test.sh -f tsim/table/tinyint.sim -./test.sh -f tsim/table/vgroup.sim - -# ---- stream -./test.sh -f tsim/stream/basic0.sim -g -# TD-20201 ./test.sh -f tsim/stream/basic1.sim -./test.sh -f tsim/stream/basic2.sim -./test.sh -f tsim/stream/drop_stream.sim -./test.sh -f tsim/stream/fillHistoryBasic1.sim -./test.sh -f tsim/stream/fillHistoryBasic2.sim -./test.sh -f tsim/stream/fillHistoryBasic3.sim -./test.sh -f tsim/stream/distributeInterval0.sim -./test.sh -f tsim/stream/distributeIntervalRetrive0.sim -./test.sh -f tsim/stream/distributeSession0.sim -./test.sh -f tsim/stream/session0.sim -./test.sh -f tsim/stream/session1.sim -./test.sh -f tsim/stream/state0.sim -./test.sh -f tsim/stream/triggerInterval0.sim -./test.sh -f tsim/stream/triggerSession0.sim -./test.sh -f tsim/stream/partitionby.sim -./test.sh -f tsim/stream/partitionby1.sim -./test.sh -f tsim/stream/schedSnode.sim -./test.sh -f tsim/stream/windowClose.sim -./test.sh -f tsim/stream/ignoreExpiredData.sim -./test.sh -f tsim/stream/sliding.sim -./test.sh -f tsim/stream/partitionbyColumnInterval.sim -./test.sh -f tsim/stream/partitionbyColumnSession.sim -./test.sh -f tsim/stream/partitionbyColumnState.sim -./test.sh -f tsim/stream/deleteInterval.sim -./test.sh -f tsim/stream/deleteSession.sim -./test.sh -f tsim/stream/deleteState.sim -./test.sh -f tsim/stream/fillIntervalDelete0.sim -./test.sh -f tsim/stream/fillIntervalDelete1.sim -./test.sh -f tsim/stream/fillIntervalLinear.sim -./test.sh -f tsim/stream/fillIntervalPartitionBy.sim -./test.sh -f tsim/stream/fillIntervalPrevNext.sim -./test.sh -f tsim/stream/fillIntervalValue.sim - -# ---- transaction ---- -./test.sh -f tsim/trans/lossdata1.sim -./test.sh -f tsim/trans/create_db.sim - -# ---- tmq -./test.sh -f tsim/tmq/basic1.sim -./test.sh -f tsim/tmq/basic2.sim -./test.sh -f tsim/tmq/basic3.sim -./test.sh -f tsim/tmq/basic4.sim -./test.sh -f tsim/tmq/basic1Of2Cons.sim -./test.sh -f tsim/tmq/basic2Of2Cons.sim -./test.sh -f tsim/tmq/basic3Of2Cons.sim -./test.sh -f tsim/tmq/basic4Of2Cons.sim -./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim -./test.sh -f tsim/tmq/topic.sim -./test.sh -f tsim/tmq/snapshot.sim -./test.sh -f tsim/tmq/snapshot1.sim - -# --- stable ---- -./test.sh -f tsim/stable/alter_comment.sim -./test.sh -f tsim/stable/alter_count.sim -./test.sh -f tsim/stable/alter_import.sim -./test.sh -f tsim/stable/alter_insert1.sim -./test.sh -f tsim/stable/alter_insert2.sim -./test.sh -f tsim/stable/alter_metrics.sim -./test.sh -f tsim/stable/column_add.sim -./test.sh -f tsim/stable/column_drop.sim -./test.sh -f tsim/stable/column_modify.sim -./test.sh -f tsim/stable/disk.sim -./test.sh -f tsim/stable/dnode3.sim -./test.sh -f tsim/stable/metrics.sim -./test.sh -f tsim/stable/refcount.sim -./test.sh -f tsim/stable/tag_add.sim -./test.sh -f tsim/stable/tag_drop.sim -./test.sh -f tsim/stable/tag_filter.sim -./test.sh -f tsim/stable/tag_modify.sim -./test.sh -f tsim/stable/tag_rename.sim -./test.sh -f tsim/stable/values.sim -./test.sh -f tsim/stable/vnode3.sim -./test.sh -f tsim/stable/metrics_idx.sim - -# --- sma -./test.sh -f tsim/sma/drop_sma.sim -./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim - -# --- valgrind ---- -./test.sh -f tsim/valgrind/checkError1.sim -./test.sh -f tsim/valgrind/checkError2.sim -./test.sh -f tsim/valgrind/checkError3.sim -./test.sh -f tsim/valgrind/checkError4.sim -./test.sh -f tsim/valgrind/checkError5.sim -./test.sh -f tsim/valgrind/checkError6.sim -./test.sh -f tsim/valgrind/checkError7.sim -./test.sh -f tsim/valgrind/checkError8.sim -./test.sh -f tsim/valgrind/checkUdf.sim - -# --- vnode ---- -./test.sh -f tsim/vnode/replica3_basic.sim -./test.sh -f tsim/vnode/replica3_repeat.sim -./test.sh -f tsim/vnode/replica3_vgroup.sim -./test.sh -f tsim/vnode/replica3_many.sim -./test.sh -f tsim/vnode/replica3_import.sim -./test.sh -f tsim/vnode/stable_balance_replica1.sim -./test.sh -f tsim/vnode/stable_dnode2_stop.sim -./test.sh -f tsim/vnode/stable_dnode2.sim -./test.sh -f tsim/vnode/stable_dnode3.sim -./test.sh -f tsim/vnode/stable_replica3_dnode6.sim -./test.sh -f tsim/vnode/stable_replica3_vnode3.sim - -# --- sync -./test.sh -f tsim/sync/3Replica1VgElect.sim -./test.sh -f tsim/sync/3Replica5VgElect.sim -./test.sh -f tsim/sync/oneReplica1VgElect.sim -./test.sh -f tsim/sync/oneReplica5VgElect.sim - -# --- catalog ---- -./test.sh -f tsim/catalog/alterInCurrent.sim - -# --- scalar ---- -./test.sh -f tsim/scalar/in.sim -./test.sh -f tsim/scalar/scalar.sim -./test.sh -f tsim/scalar/filter.sim -./test.sh -f tsim/scalar/caseWhen.sim - -# ---- alter ---- -./test.sh -f tsim/alter/cached_schema_after_alter.sim -./test.sh -f tsim/alter/dnode.sim -./test.sh -f tsim/alter/table.sim - -# ---- cache ---- -./test.sh -f tsim/cache/new_metrics.sim -./test.sh -f tsim/cache/restart_table.sim -./test.sh -f tsim/cache/restart_metrics.sim - -# ---- column ---- -./test.sh -f tsim/column/commit.sim -./test.sh -f tsim/column/metrics.sim -./test.sh -f tsim/column/table.sim - -# ---- compress ---- -./test.sh -f tsim/compress/commitlog.sim -./test.sh -f tsim/compress/compress2.sim -./test.sh -f tsim/compress/compress.sim -./test.sh -f tsim/compress/uncompress.sim - -# ---- compute ---- -./test.sh -f tsim/compute/avg.sim -./test.sh -f tsim/compute/block_dist.sim -./test.sh -f tsim/compute/bottom.sim -./test.sh -f tsim/compute/count.sim -./test.sh -f tsim/compute/diff.sim -./test.sh -f tsim/compute/diff2.sim -./test.sh -f tsim/compute/first.sim -./test.sh -f tsim/compute/interval.sim -./test.sh -f tsim/compute/last_row.sim -./test.sh -f tsim/compute/last.sim -./test.sh -f tsim/compute/leastsquare.sim -./test.sh -f tsim/compute/max.sim -./test.sh -f tsim/compute/min.sim -./test.sh -f tsim/compute/null.sim -./test.sh -f tsim/compute/percentile.sim -./test.sh -f tsim/compute/stddev.sim -./test.sh -f tsim/compute/sum.sim -./test.sh -f tsim/compute/top.sim - -# ---- field ---- -./test.sh -f tsim/field/2.sim -./test.sh -f tsim/field/3.sim -./test.sh -f tsim/field/4.sim -./test.sh -f tsim/field/5.sim -./test.sh -f tsim/field/6.sim -./test.sh -f tsim/field/binary.sim -./test.sh -f tsim/field/bigint.sim -./test.sh -f tsim/field/bool.sim -./test.sh -f tsim/field/double.sim -./test.sh -f tsim/field/float.sim -./test.sh -f tsim/field/int.sim -./test.sh -f tsim/field/single.sim -./test.sh -f tsim/field/smallint.sim -./test.sh -f tsim/field/tinyint.sim -./test.sh -f tsim/field/unsigined_bigint.sim - -# ---- vector ---- -./test.sh -f tsim/vector/metrics_field.sim -./test.sh -f tsim/vector/metrics_mix.sim -./test.sh -f tsim/vector/metrics_query.sim -./test.sh -f tsim/vector/metrics_tag.sim -./test.sh -f tsim/vector/metrics_time.sim -./test.sh -f tsim/vector/multi.sim -./test.sh -f tsim/vector/single.sim -./test.sh -f tsim/vector/table_field.sim -./test.sh -f tsim/vector/table_mix.sim -./test.sh -f tsim/vector/table_query.sim -./test.sh -f tsim/vector/table_time.sim - -# ---- wal ---- -./test.sh -f tsim/wal/kill.sim - -# ---- tag ---- -./test.sh -f tsim/tag/3.sim -./test.sh -f tsim/tag/4.sim -./test.sh -f tsim/tag/5.sim -./test.sh -f tsim/tag/6.sim -./test.sh -f tsim/tag/add.sim -./test.sh -f tsim/tag/bigint.sim -./test.sh -f tsim/tag/binary_binary.sim -./test.sh -f tsim/tag/binary.sim -./test.sh -f tsim/tag/bool_binary.sim -./test.sh -f tsim/tag/bool_int.sim -./test.sh -f tsim/tag/bool.sim -./test.sh -f tsim/tag/change.sim -./test.sh -f tsim/tag/column.sim -./test.sh -f tsim/tag/commit.sim -./test.sh -f tsim/tag/create.sim -./test.sh -f tsim/tag/delete.sim -./test.sh -f tsim/tag/double.sim -./test.sh -f tsim/tag/filter.sim -./test.sh -f tsim/tag/float.sim -./test.sh -f tsim/tag/int_binary.sim -./test.sh -f tsim/tag/int_float.sim -./test.sh -f tsim/tag/int.sim -./test.sh -f tsim/tag/set.sim -./test.sh -f tsim/tag/smallint.sim -./test.sh -f tsim/tag/tinyint.sim -./test.sh -f tsim/tag/drop_tag.sim -./test.sh -f tsim/tag/tbNameIn.sim - -./test.sh -f tmp/monitor.sim -#======================b1-end=============== diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh deleted file mode 100755 index 0729d36f12..0000000000 --- a/tests/system-test/fulltest.sh +++ /dev/null @@ -1,656 +0,0 @@ -#!/bin/bash -set -e -set -x - -python3 ./test.py -f 0-others/taosShell.py -python3 ./test.py -f 0-others/taosShellError.py -python3 ./test.py -f 0-others/taosShellNetChk.py -python3 ./test.py -f 0-others/telemetry.py -python3 ./test.py -f 0-others/taosdMonitor.py -python3 ./test.py -f 0-others/udfTest.py -python3 ./test.py -f 0-others/udf_create.py -python3 ./test.py -f 0-others/udf_restart_taosd.py -python3 ./test.py -f 0-others/cachemodel.py -python3 ./test.py -f 0-others/udf_cfg1.py -python3 ./test.py -f 0-others/udf_cfg2.py -python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 -python3 ./test.py -f 0-others/sysinfo.py -python3 ./test.py -f 0-others/user_control.py -python3 ./test.py -f 0-others/fsync.py -python3 ./test.py -f 0-others/compatibility.py -python3 ./test.py -f 1-insert/alter_database.py -python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -python3 ./test.py -f 1-insert/alter_stable.py -python3 ./test.py -f 1-insert/alter_table.py -python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -python3 ./test.py -f 1-insert/table_comment.py -python3 ./test.py -f 1-insert/time_range_wise.py -python3 ./test.py -f 1-insert/block_wise.py -python3 ./test.py -f 1-insert/create_retentions.py -python3 ./test.py -f 1-insert/table_param_ttl.py -python3 ./test.py -f 1-insert/mutil_stage.py -python3 ./test.py -f 1-insert/table_param_ttl.py -R -python3 ./test.py -f 1-insert/update_data_muti_rows.py -python3 ./test.py -f 1-insert/db_tb_name_check.py -python3 ./test.py -f 1-insert/database_pre_suf.py -python3 ./test.py -f 0-others/show.py -python3 ./test.py -f 2-query/abs.py -python3 ./test.py -f 2-query/abs.py -R -python3 ./test.py -f 2-query/and_or_for_byte.py -python3 ./test.py -f 2-query/and_or_for_byte.py -R -python3 ./test.py -f 2-query/apercentile.py -python3 ./test.py -f 2-query/apercentile.py -R -python3 ./test.py -f 2-query/arccos.py -python3 ./test.py -f 2-query/arccos.py -R -python3 ./test.py -f 2-query/arcsin.py -python3 ./test.py -f 2-query/arcsin.py -R -python3 ./test.py -f 2-query/arctan.py -python3 ./test.py -f 2-query/arctan.py -R -python3 ./test.py -f 2-query/avg.py -python3 ./test.py -f 2-query/avg.py -R -python3 ./test.py -f 2-query/between.py -python3 ./test.py -f 2-query/between.py -R -python3 ./test.py -f 2-query/bottom.py -python3 ./test.py -f 2-query/bottom.py -R -python3 ./test.py -f 2-query/cast.py -python3 ./test.py -f 2-query/cast.py -R -python3 ./test.py -f 2-query/ceil.py -python3 ./test.py -f 2-query/ceil.py -R -python3 ./test.py -f 2-query/char_length.py -python3 ./test.py -f 2-query/char_length.py -R -python3 ./test.py -f 2-query/check_tsdb.py -python3 ./test.py -f 2-query/check_tsdb.py -R -python3 ./test.py -f 2-query/concat.py -python3 ./test.py -f 2-query/concat.py -R -python3 ./test.py -f 2-query/concat_ws.py -python3 ./test.py -f 2-query/concat_ws.py -R -python3 ./test.py -f 2-query/concat_ws2.py -python3 ./test.py -f 2-query/concat_ws2.py -R -python3 ./test.py -f 2-query/cos.py -python3 ./test.py -f 2-query/cos.py -R -python3 ./test.py -f 2-query/count_partition.py -python3 ./test.py -f 2-query/count_partition.py -R -python3 ./test.py -f 2-query/count.py -python3 ./test.py -f 2-query/count.py -R -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R -python3 ./test.py -f 2-query/db.py -python3 ./test.py -f 2-query/db.py -R -python3 ./test.py -f 2-query/diff.py -python3 ./test.py -f 2-query/diff.py -R -python3 ./test.py -f 2-query/distinct.py -python3 ./test.py -f 2-query/distinct.py -R -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R -python3 ./test.py -f 2-query/distribute_agg_avg.py -python3 ./test.py -f 2-query/distribute_agg_avg.py -R -python3 ./test.py -f 2-query/distribute_agg_count.py -python3 ./test.py -f 2-query/distribute_agg_count.py -R -python3 ./test.py -f 2-query/distribute_agg_max.py -python3 ./test.py -f 2-query/distribute_agg_max.py -R -python3 ./test.py -f 2-query/distribute_agg_min.py -python3 ./test.py -f 2-query/distribute_agg_min.py -R -python3 ./test.py -f 2-query/distribute_agg_spread.py -python3 ./test.py -f 2-query/distribute_agg_spread.py -R -python3 ./test.py -f 2-query/distribute_agg_stddev.py -python3 ./test.py -f 2-query/distribute_agg_stddev.py -R -python3 ./test.py -f 2-query/distribute_agg_sum.py -python3 ./test.py -f 2-query/distribute_agg_sum.py -R -python3 ./test.py -f 2-query/explain.py -python3 ./test.py -f 2-query/explain.py -R -python3 ./test.py -f 2-query/first.py -python3 ./test.py -f 2-query/first.py -R -python3 ./test.py -f 2-query/floor.py -python3 ./test.py -f 2-query/floor.py -R -python3 ./test.py -f 2-query/function_null.py -python3 ./test.py -f 2-query/function_null.py -R -python3 ./test.py -f 2-query/function_stateduration.py -python3 ./test.py -f 2-query/function_stateduration.py -R -python3 ./test.py -f 2-query/histogram.py -python3 ./test.py -f 2-query/histogram.py -R -python3 ./test.py -f 2-query/hyperloglog.py -python3 ./test.py -f 2-query/hyperloglog.py -R -python3 ./test.py -f 2-query/interp.py -python3 ./test.py -f 2-query/interp.py -R -python3 ./test.py -f 2-query/irate.py -python3 ./test.py -f 2-query/irate.py -R -python3 ./test.py -f 2-query/join.py -python3 ./test.py -f 2-query/join.py -R -python3 ./test.py -f 2-query/last_row.py -python3 ./test.py -f 2-query/last_row.py -R -python3 ./test.py -f 2-query/last.py -python3 ./test.py -f 2-query/last.py -R -python3 ./test.py -f 2-query/leastsquares.py -python3 ./test.py -f 2-query/leastsquares.py -R -python3 ./test.py -f 2-query/length.py -python3 ./test.py -f 2-query/length.py -R -python3 ./test.py -f 2-query/log.py -python3 ./test.py -f 2-query/log.py -R -python3 ./test.py -f 2-query/lower.py -python3 ./test.py -f 2-query/lower.py -R -python3 ./test.py -f 2-query/ltrim.py -python3 ./test.py -f 2-query/ltrim.py -R -python3 ./test.py -f 2-query/mavg.py -python3 ./test.py -f 2-query/mavg.py -R -python3 ./test.py -f 2-query/max_partition.py -python3 ./test.py -f 2-query/max_partition.py -R -python3 ./test.py -f 2-query/max.py -python3 ./test.py -f 2-query/max.py -R -python3 ./test.py -f 2-query/min.py -python3 ./test.py -f 2-query/min.py -R -python3 ./test.py -f 2-query/mode.py -python3 ./test.py -f 2-query/mode.py -R -python3 ./test.py -f 2-query/Now.py -python3 ./test.py -f 2-query/Now.py -R -python3 ./test.py -f 2-query/percentile.py -python3 ./test.py -f 2-query/percentile.py -R -python3 ./test.py -f 2-query/pow.py -python3 ./test.py -f 2-query/pow.py -R -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R -python3 ./test.py -f 2-query/round.py -python3 ./test.py -f 2-query/round.py -R -python3 ./test.py -f 2-query/rtrim.py -python3 ./test.py -f 2-query/rtrim.py -R -python3 ./test.py -f 2-query/sample.py -python3 ./test.py -f 2-query/sample.py -R -python3 ./test.py -f 2-query/sin.py -python3 ./test.py -f 2-query/sin.py -R -python3 ./test.py -f 2-query/smaTest.py -python3 ./test.py -f 2-query/smaTest.py -R -python3 ./test.py -f 2-query/sml.py -python3 ./test.py -f 2-query/sml.py -R -python3 ./test.py -f 2-query/spread.py -python3 ./test.py -f 2-query/spread.py -R -python3 ./test.py -f 2-query/sqrt.py -python3 ./test.py -f 2-query/sqrt.py -R -python3 ./test.py -f 2-query/statecount.py -python3 ./test.py -f 2-query/statecount.py -R -python3 ./test.py -f 2-query/stateduration.py -python3 ./test.py -f 2-query/stateduration.py -R -python3 ./test.py -f 2-query/substr.py -python3 ./test.py -f 2-query/substr.py -R -python3 ./test.py -f 2-query/sum.py -python3 ./test.py -f 2-query/sum.py -R -python3 ./test.py -f 2-query/tail.py -python3 ./test.py -f 2-query/tail.py -R -python3 ./test.py -f 2-query/tan.py -python3 ./test.py -f 2-query/tan.py -R -python3 ./test.py -f 2-query/Timediff.py -python3 ./test.py -f 2-query/Timediff.py -R -python3 ./test.py -f 2-query/timetruncate.py -python3 ./test.py -f 2-query/timetruncate.py -R -python3 ./test.py -f 2-query/timezone.py -python3 ./test.py -f 2-query/timezone.py -R -python3 ./test.py -f 2-query/To_iso8601.py -python3 ./test.py -f 2-query/To_iso8601.py -R -python3 ./test.py -f 2-query/To_unixtimestamp.py -python3 ./test.py -f 2-query/To_unixtimestamp.py -R -python3 ./test.py -f 2-query/Today.py -python3 ./test.py -f 2-query/Today.py -R -python3 ./test.py -f 2-query/top.py -python3 ./test.py -f 2-query/top.py -R -python3 ./test.py -f 2-query/tsbsQuery.py -python3 ./test.py -f 2-query/tsbsQuery.py -R -python3 ./test.py -f 2-query/ttl_comment.py -python3 ./test.py -f 2-query/ttl_comment.py -R -python3 ./test.py -f 2-query/twa.py -python3 ./test.py -f 2-query/twa.py -R -python3 ./test.py -f 2-query/union.py -python3 ./test.py -f 2-query/union.py -R -python3 ./test.py -f 2-query/unique.py -python3 ./test.py -f 2-query/unique.py -R -python3 ./test.py -f 2-query/upper.py -python3 ./test.py -f 2-query/upper.py -R -python3 ./test.py -f 2-query/varchar.py -python3 ./test.py -f 2-query/varchar.py -R - - -python3 ./test.py -f 1-insert/update_data.py -python3 ./test.py -f 1-insert/tb_100w_data_order.py - -# TD-20200 python3 ./test.py -f 1-insert/delete_data.py -python3 ./test.py -f 1-insert/delete_stable.py -python3 ./test.py -f 1-insert/delete_childtable.py -python3 ./test.py -f 1-insert/delete_normaltable.py -python3 ./test.py -f 1-insert/keep_expired.py - -python3 ./test.py -f 2-query/join2.py -python3 ./test.py -f 2-query/union1.py -python3 ./test.py -f 2-query/concat2.py - - -python3 ./test.py -f 2-query/json_tag.py - -python3 ./test.py -f 2-query/nestedQuery.py -python3 ./test.py -f 2-query/nestedQuery_str.py -python3 ./test.py -f 2-query/nestedQuery_math.py -python3 ./test.py -f 2-query/nestedQuery_time.py -python3 ./test.py -f 2-query/stablity.py -python3 ./test.py -f 2-query/stablity_1.py - -python3 ./test.py -f 2-query/elapsed.py -python3 ./test.py -f 2-query/csum.py -python3 ./test.py -f 2-query/function_diff.py -python3 ./test.py -f 2-query/queryQnode.py - -python3 ./test.py -f 6-cluster/5dnode1mnode.py -python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False -python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False -python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 - -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 - -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 - - -python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 - -# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -# TD-19646 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 - -python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 - -# vnode case -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - -python3 ./test.py -f 7-tmq/create_wrong_topic.py -python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 -python3 ./test.py -f 7-tmq/basic5.py -python3 ./test.py -f 7-tmq/subscribeDb.py -python3 ./test.py -f 7-tmq/subscribeDb0.py -python3 ./test.py -f 7-tmq/subscribeDb1.py -python3 ./test.py -f 7-tmq/subscribeDb2.py -python3 ./test.py -f 7-tmq/subscribeDb3.py -python3 ./test.py -f 7-tmq/subscribeDb4.py -python3 ./test.py -f 7-tmq/subscribeStb.py -python3 ./test.py -f 7-tmq/subscribeStb0.py -python3 ./test.py -f 7-tmq/subscribeStb1.py -python3 ./test.py -f 7-tmq/subscribeStb2.py -python3 ./test.py -f 7-tmq/subscribeStb3.py -python3 ./test.py -f 7-tmq/subscribeStb4.py -python3 ./test.py -f 7-tmq/db.py -python3 ./test.py -f 7-tmq/tmqError.py -python3 ./test.py -f 7-tmq/schema.py -python3 ./test.py -f 7-tmq/stbFilter.py -python3 ./test.py -f 7-tmq/tmqCheckData.py -python3 ./test.py -f 7-tmq/tmqCheckData1.py -#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 -python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py -python3 ./test.py -f 7-tmq/tmqAlterSchema.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -python3 ./test.py -f 7-tmq/tmqDropStb.py -python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqUdf.py -python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py -python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py - -python3 ./test.py -f 99-TDcase/TD-19201.py - -#------------querPolicy 2----------- - -python3 ./test.py -f 2-query/between.py -Q 2 -python3 ./test.py -f 2-query/distinct.py -Q 2 -python3 ./test.py -f 2-query/varchar.py -Q 2 -python3 ./test.py -f 2-query/ltrim.py -Q 2 -python3 ./test.py -f 2-query/rtrim.py -Q 2 -python3 ./test.py -f 2-query/length.py -Q 2 -python3 ./test.py -f 2-query/char_length.py -Q 2 -python3 ./test.py -f 2-query/upper.py -Q 2 -python3 ./test.py -f 2-query/lower.py -Q 2 -python3 ./test.py -f 2-query/join.py -Q 2 -python3 ./test.py -f 2-query/join2.py -Q 2 -python3 ./test.py -f 2-query/cast.py -Q 2 -python3 ./test.py -f 2-query/substr.py -Q 2 -python3 ./test.py -f 2-query/union.py -Q 2 -python3 ./test.py -f 2-query/union1.py -Q 2 -python3 ./test.py -f 2-query/concat.py -Q 2 -python3 ./test.py -f 2-query/concat2.py -Q 2 -python3 ./test.py -f 2-query/concat_ws.py -Q 2 -python3 ./test.py -f 2-query/concat_ws2.py -Q 2 -python3 ./test.py -f 2-query/check_tsdb.py -Q 2 -python3 ./test.py -f 2-query/spread.py -Q 2 -python3 ./test.py -f 2-query/hyperloglog.py -Q 2 -python3 ./test.py -f 2-query/explain.py -Q 2 -python3 ./test.py -f 2-query/leastsquares.py -Q 2 -python3 ./test.py -f 2-query/timezone.py -Q 2 -python3 ./test.py -f 2-query/Now.py -Q 2 -python3 ./test.py -f 2-query/Today.py -Q 2 -python3 ./test.py -f 2-query/max.py -Q 2 -python3 ./test.py -f 2-query/min.py -Q 2 -python3 ./test.py -f 2-query/mode.py -Q 2 -python3 ./test.py -f 2-query/count.py -Q 2 -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 -python3 ./test.py -f 2-query/last.py -Q 2 -python3 ./test.py -f 2-query/first.py -Q 2 -python3 ./test.py -f 2-query/To_iso8601.py -Q 2 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 -python3 ./test.py -f 2-query/timetruncate.py -Q 2 -python3 ./test.py -f 2-query/diff.py -Q 2 -python3 ./test.py -f 2-query/Timediff.py -Q 2 -python3 ./test.py -f 2-query/json_tag.py -Q 2 -python3 ./test.py -f 2-query/top.py -Q 2 -python3 ./test.py -f 2-query/bottom.py -Q 2 -python3 ./test.py -f 2-query/percentile.py -Q 2 -python3 ./test.py -f 2-query/apercentile.py -Q 2 -python3 ./test.py -f 2-query/abs.py -Q 2 -python3 ./test.py -f 2-query/ceil.py -Q 2 -python3 ./test.py -f 2-query/floor.py -Q 2 -python3 ./test.py -f 2-query/round.py -Q 2 -python3 ./test.py -f 2-query/log.py -Q 2 -python3 ./test.py -f 2-query/pow.py -Q 2 -python3 ./test.py -f 2-query/sqrt.py -Q 2 -python3 ./test.py -f 2-query/sin.py -Q 2 -python3 ./test.py -f 2-query/cos.py -Q 2 -python3 ./test.py -f 2-query/tan.py -Q 2 -python3 ./test.py -f 2-query/arcsin.py -Q 2 -python3 ./test.py -f 2-query/arccos.py -Q 2 -python3 ./test.py -f 2-query/arctan.py -Q 2 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 -python3 ./test.py -f 2-query/interp.py -Q 2 - -python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -python3 ./test.py -f 2-query/stablity.py -Q 2 -python3 ./test.py -f 2-query/stablity_1.py -Q 2 - -python3 ./test.py -f 2-query/avg.py -Q 2 -python3 ./test.py -f 2-query/elapsed.py -Q 2 -python3 ./test.py -f 2-query/csum.py -Q 2 -python3 ./test.py -f 2-query/mavg.py -Q 2 -python3 ./test.py -f 2-query/sample.py -Q 2 -python3 ./test.py -f 2-query/function_diff.py -Q 2 -python3 ./test.py -f 2-query/unique.py -Q 2 -python3 ./test.py -f 2-query/stateduration.py -Q 2 -python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -python3 ./test.py -f 2-query/statecount.py -Q 2 -python3 ./test.py -f 2-query/tail.py -Q 2 -python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -python3 ./test.py -f 2-query/twa.py -Q 2 -python3 ./test.py -f 2-query/irate.py -Q 2 -python3 ./test.py -f 2-query/function_null.py -Q 2 -python3 ./test.py -f 2-query/count_partition.py -Q 2 -python3 ./test.py -f 2-query/max_partition.py -Q 2 -python3 ./test.py -f 2-query/last_row.py -Q 2 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 -python3 ./test.py -f 2-query/sml.py -Q 2 - -#------------querPolicy 3----------- -python3 ./test.py -f 2-query/between.py -Q 3 -python3 ./test.py -f 2-query/distinct.py -Q 3 -python3 ./test.py -f 2-query/varchar.py -Q 3 -python3 ./test.py -f 2-query/ltrim.py -Q 3 -python3 ./test.py -f 2-query/rtrim.py -Q 3 -python3 ./test.py -f 2-query/length.py -Q 3 -python3 ./test.py -f 2-query/char_length.py -Q 3 -python3 ./test.py -f 2-query/upper.py -Q 3 -python3 ./test.py -f 2-query/lower.py -Q 3 -python3 ./test.py -f 2-query/join.py -Q 3 -python3 ./test.py -f 2-query/join2.py -Q 3 -python3 ./test.py -f 2-query/cast.py -Q 3 -python3 ./test.py -f 2-query/substr.py -Q 3 -python3 ./test.py -f 2-query/union.py -Q 3 -python3 ./test.py -f 2-query/union1.py -Q 3 -python3 ./test.py -f 2-query/concat.py -Q 3 -python3 ./test.py -f 2-query/concat2.py -Q 3 -python3 ./test.py -f 2-query/concat_ws.py -Q 3 -python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -python3 ./test.py -f 2-query/spread.py -Q 3 -python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -python3 ./test.py -f 2-query/explain.py -Q 3 -python3 ./test.py -f 2-query/leastsquares.py -Q 3 -python3 ./test.py -f 2-query/timezone.py -Q 3 -python3 ./test.py -f 2-query/Now.py -Q 3 -python3 ./test.py -f 2-query/Today.py -Q 3 -python3 ./test.py -f 2-query/max.py -Q 3 -python3 ./test.py -f 2-query/min.py -Q 3 -python3 ./test.py -f 2-query/mode.py -Q 3 -python3 ./test.py -f 2-query/count.py -Q 3 -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 -python3 ./test.py -f 2-query/last.py -Q 3 -python3 ./test.py -f 2-query/first.py -Q 3 -python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -python3 ./test.py -f 2-query/timetruncate.py -Q 3 -python3 ./test.py -f 2-query/diff.py -Q 3 -python3 ./test.py -f 2-query/Timediff.py -Q 3 -python3 ./test.py -f 2-query/json_tag.py -Q 3 -python3 ./test.py -f 2-query/top.py -Q 3 -python3 ./test.py -f 2-query/bottom.py -Q 3 -python3 ./test.py -f 2-query/percentile.py -Q 3 -python3 ./test.py -f 2-query/apercentile.py -Q 3 -python3 ./test.py -f 2-query/abs.py -Q 3 -python3 ./test.py -f 2-query/ceil.py -Q 3 -python3 ./test.py -f 2-query/floor.py -Q 3 -python3 ./test.py -f 2-query/round.py -Q 3 -python3 ./test.py -f 2-query/log.py -Q 3 -python3 ./test.py -f 2-query/pow.py -Q 3 -python3 ./test.py -f 2-query/sqrt.py -Q 3 -python3 ./test.py -f 2-query/sin.py -Q 3 -python3 ./test.py -f 2-query/cos.py -Q 3 -python3 ./test.py -f 2-query/tan.py -Q 3 -python3 ./test.py -f 2-query/arcsin.py -Q 3 -python3 ./test.py -f 2-query/arccos.py -Q 3 -python3 ./test.py -f 2-query/arctan.py -Q 3 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 - -python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -python3 ./test.py -f 2-query/stablity.py -Q 3 -python3 ./test.py -f 2-query/stablity_1.py -Q 3 - -python3 ./test.py -f 2-query/avg.py -Q 3 -python3 ./test.py -f 2-query/elapsed.py -Q 3 -python3 ./test.py -f 2-query/csum.py -Q 3 -python3 ./test.py -f 2-query/mavg.py -Q 3 -python3 ./test.py -f 2-query/sample.py -Q 3 -python3 ./test.py -f 2-query/function_diff.py -Q 3 -python3 ./test.py -f 2-query/unique.py -Q 3 -python3 ./test.py -f 2-query/stateduration.py -Q 3 -python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -python3 ./test.py -f 2-query/statecount.py -Q 3 -python3 ./test.py -f 2-query/tail.py -Q 3 -python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -python3 ./test.py -f 2-query/twa.py -Q 3 -python3 ./test.py -f 2-query/irate.py -Q 3 -python3 ./test.py -f 2-query/function_null.py -Q 3 -python3 ./test.py -f 2-query/count_partition.py -Q 3 -python3 ./test.py -f 2-query/max_partition.py -Q 3 -python3 ./test.py -f 2-query/last_row.py -Q 3 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -python3 ./test.py -f 2-query/sml.py -Q 3 -python3 ./test.py -f 2-query/interp.py -Q 3 - - -#------------querPolicy 4----------- - -python3 ./test.py -f 2-query/between.py -Q 4 -python3 ./test.py -f 2-query/distinct.py -Q 4 -python3 ./test.py -f 2-query/varchar.py -Q 4 -python3 ./test.py -f 2-query/ltrim.py -Q 4 -python3 ./test.py -f 2-query/rtrim.py -Q 4 -python3 ./test.py -f 2-query/length.py -Q 4 -python3 ./test.py -f 2-query/char_length.py -Q 4 -python3 ./test.py -f 2-query/upper.py -Q 4 -python3 ./test.py -f 2-query/lower.py -Q 4 -python3 ./test.py -f 2-query/join.py -Q 4 -python3 ./test.py -f 2-query/join2.py -Q 4 -#python3 ./test.py -f 2-query/cast.py -Q 4 -python3 ./test.py -f 2-query/substr.py -Q 4 -python3 ./test.py -f 2-query/union.py -Q 4 -python3 ./test.py -f 2-query/union1.py -Q 4 -python3 ./test.py -f 2-query/concat.py -Q 4 -python3 ./test.py -f 2-query/concat2.py -Q 4 -python3 ./test.py -f 2-query/concat_ws.py -Q 4 -python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -python3 ./test.py -f 2-query/spread.py -Q 4 -python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -python3 ./test.py -f 2-query/explain.py -Q 4 -python3 ./test.py -f 2-query/leastsquares.py -Q 4 -python3 ./test.py -f 2-query/timezone.py -Q 4 -python3 ./test.py -f 2-query/Now.py -Q 4 -python3 ./test.py -f 2-query/Today.py -Q 4 -python3 ./test.py -f 2-query/max.py -Q 4 -python3 ./test.py -f 2-query/min.py -Q 4 -python3 ./test.py -f 2-query/mode.py -Q 4 -python3 ./test.py -f 2-query/count.py -Q 4 -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 -python3 ./test.py -f 2-query/last.py -Q 4 -python3 ./test.py -f 2-query/first.py -Q 4 -python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -python3 ./test.py -f 2-query/timetruncate.py -Q 4 -python3 ./test.py -f 2-query/diff.py -Q 4 -python3 ./test.py -f 2-query/Timediff.py -Q 4 -python3 ./test.py -f 2-query/json_tag.py -Q 4 -python3 ./test.py -f 2-query/top.py -Q 4 -python3 ./test.py -f 2-query/bottom.py -Q 4 -python3 ./test.py -f 2-query/percentile.py -Q 4 -python3 ./test.py -f 2-query/apercentile.py -Q 4 -python3 ./test.py -f 2-query/abs.py -Q 4 -python3 ./test.py -f 2-query/ceil.py -Q 4 -python3 ./test.py -f 2-query/floor.py -Q 4 -python3 ./test.py -f 2-query/round.py -Q 4 -python3 ./test.py -f 2-query/log.py -Q 4 -python3 ./test.py -f 2-query/pow.py -Q 4 -python3 ./test.py -f 2-query/sqrt.py -Q 4 -python3 ./test.py -f 2-query/sin.py -Q 4 -python3 ./test.py -f 2-query/cos.py -Q 4 -python3 ./test.py -f 2-query/tan.py -Q 4 -python3 ./test.py -f 2-query/arcsin.py -Q 4 -python3 ./test.py -f 2-query/arccos.py -Q 4 -python3 ./test.py -f 2-query/arctan.py -Q 4 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 - -python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -python3 ./test.py -f 2-query/stablity.py -Q 4 -python3 ./test.py -f 2-query/stablity_1.py -Q 4 - - -python3 ./test.py -f 2-query/avg.py -Q 4 -python3 ./test.py -f 2-query/elapsed.py -Q 4 -python3 ./test.py -f 2-query/csum.py -Q 4 -python3 ./test.py -f 2-query/mavg.py -Q 4 -python3 ./test.py -f 2-query/sample.py -Q 4 -python3 ./test.py -f 2-query/function_diff.py -Q 4 -python3 ./test.py -f 2-query/unique.py -Q 4 -python3 ./test.py -f 2-query/stateduration.py -Q 4 -python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -python3 ./test.py -f 2-query/statecount.py -Q 4 -python3 ./test.py -f 2-query/tail.py -Q 4 -python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -python3 ./test.py -f 2-query/twa.py -Q 4 -python3 ./test.py -f 2-query/irate.py -Q 4 -python3 ./test.py -f 2-query/function_null.py -Q 4 -python3 ./test.py -f 2-query/count_partition.py -Q 4 -python3 ./test.py -f 2-query/max_partition.py -Q 4 -python3 ./test.py -f 2-query/last_row.py -Q 4 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 -python3 ./test.py -f 2-query/sml.py -Q 4 -python3 ./test.py -f 2-query/interp.py -Q 4 From 2007c0a7bd62aac4a761de404f468e6d4a60f824 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 9 Nov 2022 20:50:53 +0800 Subject: [PATCH 34/56] ci:add the env that builds with sanitizer --- tests/parallel_test/container_build.sh | 4 ++-- tests/parallel_test/run.sh | 2 +- tests/parallel_test/run_container.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 2257ba6829..be8ca69254 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -51,13 +51,13 @@ else REP_REAL_PATH=$WORKDIR/TDinternal REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal fi - +date docker run \ -v $REP_MOUNT_PARAM \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT" mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan - +date docker run \ -v $REP_MOUNT_PARAM \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT" diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index e7bab707ef..efbe83be61 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -192,7 +192,7 @@ function run_thread() { if [ ! -z "$case_path" ]; then mkdir -p $log_dir/$case_path fi - cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param} -s ${case_build_san}" + cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index 9083510d10..d93a0f9e96 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -34,7 +34,7 @@ while getopts "w:d:c:t:o:e:sh" opt; do extra_param="-o $OPTARG" ;; s) - buildSan="-o $OPTARG" + buildSan=$OPTARG ;; h) usage From 5c59e696079f1cc302270eb35349aa3426a1a1ba Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 9 Nov 2022 21:17:15 +0800 Subject: [PATCH 35/56] ci:add the env that builds with sanitizer --- tests/parallel_test/run.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index efbe83be61..fee85f0425 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -165,6 +165,14 @@ function run_thread() { case_redo_time=${DEFAULT_RETRY_TIME:-2} fi local case_build_san=`echo "$line"|cut -d, -f3` + if [ "${case_build_san}" == "y" ]; then + case_build_san="y" + elif [[ "${case_build_san}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then + case_build_san="n" + else + usage + exit 1 + fi local exec_dir=`echo "$line"|cut -d, -f4` local case_cmd=`echo "$line"|cut -d, -f5` local case_file="" @@ -193,7 +201,7 @@ function run_thread() { mkdir -p $log_dir/$case_path fi cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" - # echo "$thread_no $count $cmd" + echo "$thread_no $count $cmd" local ret=0 local redo_count=1 local case_log_file=$log_dir/${case_file}.txt From a2cec238ec255d1e423a408e7acb334c6f5923b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 22:16:56 +0800 Subject: [PATCH 36/56] fix(query): set correct operator type. --- source/libs/executor/src/groupoperator.c | 2 +- tests/script/tsim/parser/having_child.sim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 34398f9672..e31c8f588f 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -438,7 +438,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } initResultRowInfo(&pInfo->binfo.resultRowInfo); - setOperatorInfo(pOperator, "GroupbyAggOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); diff --git a/tests/script/tsim/parser/having_child.sim b/tests/script/tsim/parser/having_child.sim index ae78c806ca..db9a25365e 100644 --- a/tests/script/tsim/parser/having_child.sim +++ b/tests/script/tsim/parser/having_child.sim @@ -733,6 +733,7 @@ sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 group by f1 having sum(f1) > 2 order by f1; if $rows != 3 then + print expect 3 , actual: $rows return -1 endi if $data00 != 2.000000000 then From bd2f86f22c549474c81e3834cd8fdc70043f5a20 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 9 Nov 2022 22:22:55 +0800 Subject: [PATCH 37/56] ci:add the env that builds with sanitizer --- tests/parallel_test/cases.task | 4 ++-- tests/parallel_test/run_container.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3d3eca4e6d..f65156f04c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1,6 +1,6 @@ ,,y,unit-test,bash test.sh -,,,script,./test.sh -f tsim/user/basic.sim -,,,script,./test.sh -f tsim/user/password.sim +,,y,script,./test.sh -f tsim/user/basic.sim +,,y,script,./test.sh -f tsim/user/password.sim ,,,script,./test.sh -f tsim/user/privilege_db.sim ,,,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,,script,./test.sh -f tsim/db/alter_option.sim diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index d93a0f9e96..eac1dc41c8 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -13,7 +13,7 @@ function usage() { } ent=0 -while getopts "w:d:c:t:o:e:sh" opt; do +while getopts "w:d:c:t:o:s:eh" opt; do case $opt in w) WORKDIR=$OPTARG @@ -67,7 +67,7 @@ fi #select whether the compilation environment includes sanitizer if [ "${buildSan}" == "y" ]; then - DEBUGPATH="buildSan" + DEBUGPATH="debugSan" elif [[ "${buildSan}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then DEBUGPATH="debugNoSan" else @@ -80,7 +80,7 @@ if [ $ent -ne 0 ]; then extra_param="$extra_param -e" INTERNAL_REPDIR=$WORKDIR/TDinternal REPDIR=$INTERNAL_REPDIR/community - REPDIR_DEBUG=$WORKDIR/DEBUGPATH/ + REPDIR_DEBUG=$WORKDIR/$DEBUGPATH/ CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" @@ -89,7 +89,7 @@ if [ $ent -ne 0 ]; then else # community edition REPDIR=$WORKDIR/TDengine - REPDIR_DEBUG=$WORKDIR/DEBUGPATH/ + REPDIR_DEBUG=$WORKDIR/$DEBUGPATH/ CONTAINER_TESTDIR=/home/TDengine SIM_DIR=/home/TDengine/sim REP_MOUNT_PARAM="$REPDIR:/home/TDengine" From aedeecb81be2f5fcdd1c3d748b494a389800bbb3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 9 Nov 2022 22:42:02 +0800 Subject: [PATCH 38/56] ci:add the env that builds with sanitizer --- tests/parallel_test/container_build.sh | 6 ++++++ tests/parallel_test/run.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index be8ca69254..284b47d91a 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -56,6 +56,12 @@ docker run \ -v $REP_MOUNT_PARAM \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT" +if [[ -d ${WORKDIR}/debugNoSan ]] ;then + rm -rf ${WORKDIR}/debugNoSan +elif [[ -d ${WORKDIR}/debugSan ]] ;then + rm -rf ${WORKDIR}/debugSan +fi + mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date docker run \ diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index fee85f0425..0583f01c08 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -201,7 +201,7 @@ function run_thread() { mkdir -p $log_dir/$case_path fi cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" - echo "$thread_no $count $cmd" + # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 local case_log_file=$log_dir/${case_file}.txt From 8d6f129154fb2fc44ca59261ac52eabe8dd3ccf7 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Thu, 10 Nov 2022 08:14:21 +0800 Subject: [PATCH 39/56] fix: skip empty data in delete file --- source/dnode/vnode/src/tsdb/tsdbCache.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 76236e5078..291a5ab1eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -475,7 +475,7 @@ _err: } return code; } - +/* static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t uid, SDelIdx *pDelIdx) { int32_t code = 0; SArray *pDelIdxArray = NULL; @@ -499,7 +499,7 @@ _err: } return code; } - +*/ typedef enum { SFSLASTNEXTROW_FS, SFSLASTNEXTROW_FILESET, @@ -997,8 +997,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); - SDelIdx delIdx; - SDelFile *pDelFile = pReadSnap->fs.pDelFile; if (pDelFile) { SDelFReader *pDelFReader; @@ -1006,18 +1004,20 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code) goto _err; - code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); + SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); + + code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); + if (code) goto _err; + + SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); + + code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); if (code) { tsdbDelFReaderClose(&pDelFReader); goto _err; } + taosArrayDestroy(pDelIdxArray); tsdbDelFReaderClose(&pDelFReader); } else { code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline); From 04c6d7f26f7de23d30adf7ba1a2423d8730e68ad Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 09:10:40 +0800 Subject: [PATCH 40/56] ci:add the env that builds with sanitizer --- tests/parallel_test/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 0583f01c08..3460095b6d 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -208,7 +208,7 @@ function run_thread() { start_time=`date +%s` local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 3,4` + local case_info=`echo "$line"|cut -d, -f 4,5` while [ ${redo_count} -lt 6 ]; do if [ -f $case_log_file ]; then cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt @@ -422,7 +422,7 @@ if [ -f "${failed_case_file}" ]; then continue fi fi - line=`echo "$line"|cut -d, -f 3,4` + line=`echo "$line"|cut -d, -f 4,5` echo -e "$i. $line \e[31m failed\e[0m" >&2 i=$(( i + 1 )) done <${failed_case_file} From 14a499c22b28c370b55d6032148b85078b9227a6 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 09:58:13 +0800 Subject: [PATCH 41/56] ci:add the env that builds with sanitizer --- tests/parallel_test/container_build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 284b47d91a..4e06643ac4 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -57,8 +57,11 @@ docker run \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT" if [[ -d ${WORKDIR}/debugNoSan ]] ;then + echo "delete ${WORKDIR}/debugNoSan" rm -rf ${WORKDIR}/debugNoSan -elif [[ -d ${WORKDIR}/debugSan ]] ;then +fi +if [[ -d ${WORKDIR}/debugSan ]] ;then + echo "delete ${WORKDIR}/debugSan" rm -rf ${WORKDIR}/debugSan fi From d56dc950e46ab7a87b90fae3baed05e9c5013d37 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 10 Nov 2022 10:02:45 +0800 Subject: [PATCH 42/56] fix(stream): partition tbname reset to null --- source/libs/executor/src/groupoperator.c | 5 ++- source/libs/executor/src/scanoperator.c | 54 ++++++++++++------------ 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 891eb6e7a4..46a8baa682 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -909,7 +909,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { void* pData = colDataGetVarData(pCol, 0); // TODO check tbname validity if (pData != (void*)-1) { - memcpy(pDest->info.parTbName, varDataVal(pData), varDataLen(pData)); + memset(pDest->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN); + memcpy(pDest->info.parTbName, varDataVal(pData), len); + /*pDest->info.parTbName[len + 1] = 0;*/ } else { pDest->info.parTbName[0] = 0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6d13048123..34d6b3016d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,7 +284,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* } static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsAgg, int32_t numOfCols, - int32_t numOfRows) { + int32_t numOfRows) { if (pColsAgg == NULL || pFilterInfo == NULL) { return true; } @@ -345,7 +345,7 @@ static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlo // todo handle the slimit info void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator) { - SLimit* pLimit = &pLimitInfo->limit; + SLimit* pLimit = &pLimitInfo->limit; const char* id = GET_TASKID(pTaskInfo); if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) { @@ -499,7 +499,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction typedef struct STableCachedVal { const char* pName; - STag* pTags; + STag* pTags; } STableCachedVal; static void freeTableCachedVal(void* param) { @@ -513,13 +513,11 @@ static void freeTableCachedVal(void* param) { taosMemoryFree(pVal); } -//const void *key, size_t keyLen, void *value -static void freeCachedMetaItem(const void *key, size_t keyLen, void *value) { - freeTableCachedVal(value); -} +// const void *key, size_t keyLen, void *value +static void freeCachedMetaItem(const void* key, size_t keyLen, void* value) { freeTableCachedVal(value); } -int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, - SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo* pCache) { +int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, + int32_t rows, const char* idStr, STableMetaCacheInfo* pCache) { // currently only the tbname pseudo column if (numOfExpr <= 0) { return TSDB_CODE_SUCCESS; @@ -531,11 +529,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int int32_t backupRows = pBlock->info.rows; pBlock->info.rows = rows; - bool freeReader = false; + bool freeReader = false; STableCachedVal val = {0}; SMetaReader mr = {0}; - LRUHandle* h = NULL; + LRUHandle* h = NULL; // 1. check if it is existed in meta cache if (pCache == NULL) { @@ -582,7 +580,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int val = *pVal; freeReader = true; - int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal, sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); + int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal, + sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); if (ret != TAOS_LRU_STATUS_OK) { qError("failed to put meta into lru cache, code:%d, %s", ret, idStr); freeTableCachedVal(pVal); @@ -594,13 +593,13 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int taosLRUCacheRelease(pCache->pTableMetaEntryCache, h, false); } - qDebug("retrieve table meta from cache:%"PRIu64", hit:%"PRIu64 " miss:%"PRIu64", %s", pCache->metaFetch, pCache->cacheHit, - (pCache->metaFetch - pCache->cacheHit), idStr); + qDebug("retrieve table meta from cache:%" PRIu64 ", hit:%" PRIu64 " miss:%" PRIu64 ", %s", pCache->metaFetch, + pCache->cacheHit, (pCache->metaFetch - pCache->cacheHit), idStr); } for (int32_t j = 0; j < numOfExpr; ++j) { const SExprInfo* pExpr1 = &pExpr[j]; - int32_t dstSlotId = pExpr1->base.resSchema.slotId; + int32_t dstSlotId = pExpr1->base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); colInfoDataCleanup(pColInfoData, pBlock->info.rows); @@ -652,7 +651,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, fmGetScalarFuncExecFuncs(functionId, &fpSet); size_t len = TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE; - char buf[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char buf[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(buf, name) SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, len, 1); @@ -904,12 +903,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - SScanPhysiNode* pScanNode = &pTableScanNode->scan; + SScanPhysiNode* pScanNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, - &pInfo->matchInfo); + int32_t code = + extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -955,7 +954,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024*128, -1, .5); + pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->metaCache.pTableMetaEntryCache == NULL) { code = terrno; goto _error; @@ -1049,8 +1048,8 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, - GET_TASKID(pTaskInfo)); + int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, + (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -1599,8 +1598,10 @@ static void calBlockTbName(SExprSupp* pTbNameCalSup, SSDataBlock* pBlock) { void* pData = colDataGetData(pCol, 0); // TODO check tbname validation if (pData != (void*)-1 && pData != NULL) { - memcpy(pBlock->info.parTbName, varDataVal(pData), TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN)); - pBlock->info.parTbName[TSDB_TABLE_NAME_LEN - 1] = 0; + memset(pBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN); + memcpy(pBlock->info.parTbName, varDataVal(pData), len); + /*pBlock->info.parTbName[len + 1] = 0;*/ } else { pBlock->info.parTbName[0] = 0; } @@ -4319,7 +4320,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4455,7 +4456,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc T_LONG_JMP(pTaskInfo->env, code); } - if (pOperator->exprSupp.pFilterInfo!= NULL) { + if (pOperator->exprSupp.pFilterInfo != NULL) { int64_t st = taosGetTimestampMs(); doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo); @@ -4831,7 +4832,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->sample.seed = taosGetTimestampSec(); pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; - code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { goto _error; From 962bbb204d6384c01b97f6c3db51c301301c2661 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 10:48:58 +0800 Subject: [PATCH 43/56] ci:add the env that builds with sanitizer --- tests/parallel_test/run.sh | 4 ++-- tests/parallel_test/run_container.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 3460095b6d..b5d57265be 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -208,7 +208,7 @@ function run_thread() { start_time=`date +%s` local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 4,5` + local case_info=`echo "$line"|cut -d, -f 3,4,5` while [ ${redo_count} -lt 6 ]; do if [ -f $case_log_file ]; then cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt @@ -422,7 +422,7 @@ if [ -f "${failed_case_file}" ]; then continue fi fi - line=`echo "$line"|cut -d, -f 4,5` + line=`echo "$line"|cut -d, -f 3,4,5` echo -e "$i. $line \e[31m failed\e[0m" >&2 i=$(( i + 1 )) done <${failed_case_file} diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index eac1dc41c8..9d76f40193 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -84,7 +84,7 @@ if [ $ent -ne 0 ]; then CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" - REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/:ro" else # community edition From f7d263862b3d8b508cd114669def31432ba3045e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 10:04:05 +0800 Subject: [PATCH 44/56] refact: remove assert and adjust log --- source/libs/sync/inc/syncSnapshot.h | 14 +- source/libs/sync/inc/syncTimeout.h | 1 - source/libs/sync/inc/syncTools.h | 1 - source/libs/sync/inc/syncUtil.h | 8 +- source/libs/sync/inc/syncVoteMgr.h | 4 - source/libs/sync/src/syncMain.c | 16 +- source/libs/sync/src/syncSnapshot.c | 136 +---------------- source/libs/sync/src/syncTimeout.c | 27 ++-- source/libs/sync/src/syncUtil.c | 32 +--- source/libs/sync/src/syncVoteMgr.c | 140 ++++-------------- .../sync/test/sync_test_lib/inc/syncTest.h | 12 ++ .../sync_test_lib/src/syncSnapshotDebug.c | 132 +++++++++++++++++ .../test/sync_test_lib/src/syncUtilDebug.c | 36 +++++ .../test/sync_test_lib/src/syncVoteMgrDebug.c | 102 +++++++++++++ 14 files changed, 349 insertions(+), 312 deletions(-) create mode 100644 source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index d688a3c1bd..963fedce31 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -21,7 +21,6 @@ extern "C" { #endif #include "syncInt.h" -#include "syncMessage.h" #define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 @@ -31,7 +30,6 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 -//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; @@ -60,12 +58,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finis int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); - -int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); - -//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; @@ -78,7 +70,6 @@ typedef struct SSyncSnapshotReceiver { // init when create SSyncNode *pSyncNode; - } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); @@ -88,13 +79,10 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); - -//--------------------------------------------------- // on message int32_t syncNodeOnSnapshot(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotReply(SSyncNode *ths, SyncSnapshotRsp *pMsg); +int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index e1a6050d92..3139707d54 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -21,7 +21,6 @@ extern "C" { #endif #include "syncInt.h" -#include "syncMessage.h" // TLA+ Spec // Timeout(i) == /\ state[i] \in {Follower, Candidate} diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 6a760ecd87..3fb4a5ba0c 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -728,7 +728,6 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg); int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); -int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg); int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg); // ----------------------------------------- diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 7f241e827d..076101ef43 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -24,17 +24,15 @@ extern "C" { uint64_t syncUtilAddr2U64(const char* host, uint16_t port); void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port); -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); bool syncUtilEmptyId(const SRaftId* pId); int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); -char* syncUtilRaftId2Str(const SRaftId* p); const char* syncStr(ESyncState state); char* syncUtilPrintBin(char* ptr, uint32_t len); char* syncUtilPrintBin2(char* ptr, uint32_t len); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index d894e91600..066a4dd76f 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -39,8 +39,6 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncN bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted); -char *voteGranted2Str(SVotesGranted *pVotesGranted); typedef struct SVotesRespond { SRaftId (*replicas)[TSDB_MAX_REPLICA]; @@ -56,8 +54,6 @@ void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSync bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond); -char *votesRespond2Str(SVotesRespond *pVotesRespond); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 252d5d0219..25f6c5e6f1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -757,7 +757,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - if (!syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId); goto _error; } @@ -772,7 +772,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { sError("vgId:%d, failed to determine raft member id, peer:%d", pSyncNode->vgId, i); goto _error; } @@ -781,7 +781,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; } @@ -1213,7 +1213,7 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilraftId2EpSet(destRaftId, &epSet); + syncUtilRaftId2EpSet(destRaftId, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1230,7 +1230,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilnodeInfo2EpSet(nodeInfo, &epSet); + syncUtilNodeInfo2EpSet(nodeInfo, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1344,7 +1344,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); + syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; @@ -1356,13 +1356,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); } // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } // update quorum first diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 78413bbeff..3273718f7d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -13,33 +13,25 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncSnapshot.h" #include "syncIndexMgr.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" -#include "wal.h" -//---------------------------------- -static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg); -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); -static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); - -//---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && (pSyncNode->pFsm->FpSnapshotDoRead != NULL); SSyncSnapshotSender *pSender = NULL; if (condition) { - pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + pSender = taosMemoryCalloc(1, sizeof(SSyncSnapshotSender)); if (pSender == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pSender, 0, sizeof(*pSender)); pSender->start = false; pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; @@ -249,64 +241,6 @@ static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnaps ++(pSender->seq); } -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pSender != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pSender->start); - cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); - cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); - cJSON_AddStringToObject(pRoot, "pReader", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); - cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); - - if (pSender->pCurrentBlock != NULL) { - char *s; - s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); - taosMemoryFree(s); - } - - cJSON *pSnapshot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); - cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); - cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); - // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); - return pJson; -} - -char *snapshotSender2Str(SSyncSnapshotSender *pSender) { - cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { sNTrace(pSyncNode, "starting snapshot ..."); @@ -335,16 +269,17 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } -// ------------------------------------- SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); SSyncSnapshotReceiver *pReceiver = NULL; if (condition) { - pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); - ASSERT(pReceiver != NULL); - memset(pReceiver, 0, sizeof(*pReceiver)); + pReceiver = taosMemoryCalloc(1, sizeof(SSyncSnapshotReceiver)); + if (pReceiver == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pReceiver->start = false; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; @@ -530,63 +465,6 @@ static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapsh } } -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pReceiver != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); - cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); - cJSON_AddStringToObject(pRoot, "pWriter", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - cJSON *pFromId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); - cJSON_AddStringToObject(pFromId, "addr", u64buf); - { - uint64_t u64 = pReceiver->fromId.addr; - cJSON *pTmp = pFromId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); - cJSON_AddItemToObject(pRoot, "fromId", pFromId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); - return pJson; -} - -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { - cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { SyncIndex snapStart = SYNC_INDEX_INVALID; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 30e0617f43..91d807319b 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -13,28 +13,23 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncTimeout.h" #include "syncElection.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncReplication.h" -#include "syncRespMgr.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { int32_t newArrIndex = 0; - SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT]; - memset(newConfigIndexArr, 0, sizeof(newConfigIndexArr)); - + SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT] = {0}; SSnapshot snapshot = {0}; - if (ths->pFsm != NULL && ths->pFsm->FpGetSnapshotInfo != NULL) { - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - } - + + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); if (snapshot.lastApplyIndex != SYNC_INDEX_INVALID) { - for (int i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { + for (int32_t i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { if (ths->pRaftCfg->configIndexArr[i] < snapshot.lastConfigIndex) { // pass - ; } else { // save newConfigIndexArr[newArrIndex] = ths->pRaftCfg->configIndexArr[i]; @@ -47,13 +42,15 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { memcpy(ths->pRaftCfg->configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr)); int32_t code = raftCfgPersist(ths->pRaftCfg); - ASSERT(code == 0); - - sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + if (code != 0) { + sNFatal(ths, "failed to persist cfg"); + } else { + sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + } } } -int32_t syncNodeTimerRoutine(SSyncNode* ths) { +static int32_t syncNodeTimerRoutine(SSyncNode* ths) { sNTrace(ths, "timer routines"); // timer replicate @@ -71,7 +68,7 @@ int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncLogStoreData* pData = ths->pLogStore->data; int32_t code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, timer wal snapshot end error since:%s", ths->vgId, terrstr()); + sNError(ths, "timer wal snapshot end error since:%s", terrstr()); return -1; } else { sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 1a00b0f5a4..894aa05aad 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -41,15 +41,15 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { pEpSet->inUse = 0; pEpSet->numOfEps = 0; addEpIntoEpSet(pEpSet, pInfo->nodeFqdn, pInfo->nodePort); } -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { char host[TSDB_FQDN_LEN] = {0}; - uint16_t port; + uint16_t port = 0; syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); pEpSet->inUse = 0; @@ -57,7 +57,7 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { addEpIntoEpSet(pEpSet, host, port); } -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { sError("failed to resolve ipv4 addr, fqdn: %s", pInfo->nodeFqdn); @@ -73,8 +73,7 @@ bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { - bool ret = pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; - return ret; + return pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; } bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } @@ -90,18 +89,6 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); - cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); - return pJson; -} - cJSON* syncUtilRaftId2Json(const SRaftId* p) { char u64buf[128] = {0}; cJSON* pRoot = cJSON_CreateObject(); @@ -120,13 +107,6 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { return pJson; } -char* syncUtilRaftId2Str(const SRaftId* p) { - cJSON* pJson = syncUtilRaftId2Json(p); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - static inline bool syncUtilCanPrint(char c) { if (c >= 32 && c <= 126) { return true; @@ -165,14 +145,12 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) { } void syncUtilMsgHtoN(void* msg) { - // htonl SMsgHead* pHead = msg; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); } void syncUtilMsgNtoH(void* msg) { - // ntohl SMsgHead* pHead = msg; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index ee1f83ee6a..b36e5c0288 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,12 +23,11 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryCalloc(1, sizeof(SVotesGranted)); if (pVotesGranted == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pVotesGranted, 0, sizeof(SVotesGranted)); pVotesGranted->replicas = &(pSyncNode->replicasId); pVotesGranted->replicaNum = pSyncNode->replicaNum; @@ -59,20 +58,24 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode) { pVotesGranted->pSyncNode = pSyncNode; } -bool voteGrantedMajority(SVotesGranted *pVotesGranted) { - bool ret = pVotesGranted->votes >= pVotesGranted->quorum; - return ret; -} +bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->votes >= pVotesGranted->quorum; } void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { - ASSERT(pMsg->voteGranted == true); - - if (pMsg->term != pVotesGranted->term) { - sNTrace(pVotesGranted->pSyncNode, "vote grant vnode error"); + if (!pMsg->voteGranted) { + sNFatal(pVotesGranted->pSyncNode, " vote granted should be true"); return; } - ASSERT(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + if (pMsg->term != pVotesGranted->term) { + sNTrace(pVotesGranted->pSyncNode, "vote grant term:%" PRId64 " not matched with msg term:%" PRId64, + pVotesGranted->term, pMsg->term); + return; + } + + if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { + sNFatal(pVotesGranted->pSyncNode, " vote granted raftId not matched with msg"); + return; + } int32_t j = -1; for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -81,14 +84,21 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - ASSERT(j != -1); - ASSERT(j >= 0 && j < pVotesGranted->replicaNum); + if ((j == -1) || (j >= 0 && j < pVotesGranted->replicaNum)) { + sNFatal(pVotesGranted->pSyncNode, " invalid msg srcId, index:%d", j); + return; + } if (pVotesGranted->isGranted[j] != true) { ++(pVotesGranted->votes); pVotesGranted->isGranted[j] = true; } - ASSERT(pVotesGranted->votes <= pVotesGranted->replicaNum); + + if (pVotesGranted->votes <= pVotesGranted->replicaNum) { + sNFatal(pVotesGranted->pSyncNode, " votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + pVotesGranted->replicaNum); + return; + } } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { @@ -97,53 +107,12 @@ void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->toLeader = false; } -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesGranted != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); - } - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - arr[i] = pVotesGranted->isGranted[i]; - } - cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); - - cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); - cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - bool majority = voteGrantedMajority(pVotesGranted); - cJSON_AddNumberToObject(pRoot, "majority", majority); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); - return pJson; -} - -char *voteGranted2Str(SVotesGranted *pVotesGranted) { - cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); - ASSERT(pVotesRespond != NULL); - memset(pVotesRespond, 0, sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryCalloc(1, sizeof(SVotesRespond)); + if (pVotesRespond == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pVotesRespond->replicas = &(pSyncNode->replicasId); pVotesRespond->replicaNum = pSyncNode->replicaNum; @@ -185,62 +154,15 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { - // ASSERT(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } } - ASSERT(0); + + sNFatal(pVotesRespond->pSyncNode, "votes respond not found"); } void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { pVotesRespond->term = term; memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond)); - /* - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - pVotesRespond->isRespond[i] = false; - } - */ -} - -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesRespond != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); - } - int32_t respondNum = 0; - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - arr[i] = pVotesRespond->isRespond[i]; - if (pVotesRespond->isRespond[i]) { - respondNum++; - } - } - cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); - cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); - return pJson; -} - -char *votesRespond2Str(SVotesRespond *pVotesRespond) { - cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; } diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h index 3a49074272..241a4a5e02 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncTest.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -82,6 +82,18 @@ void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); +cJSON* voteGranted2Json(SVotesGranted* pVotesGranted); +char* voteGranted2Str(SVotesGranted* pVotesGranted); +cJSON* votesRespond2Json(SVotesRespond* pVotesRespond); +char* votesRespond2Str(SVotesRespond* pVotesRespond); + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); +char* syncUtilRaftId2Str(const SRaftId* p); + +cJSON* snapshotSender2Json(SSyncSnapshotSender* pSender); +char* snapshotSender2Str(SSyncSnapshotSender* pSender); +cJSON* snapshotReceiver2Json(SSyncSnapshotReceiver* pReceiver); +char* snapshotReceiver2Str(SSyncSnapshotReceiver* pReceiver); #ifdef __cplusplus } diff --git a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c new file mode 100644 index 0000000000..fdaf4b8b34 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSender != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pSender->start); + cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); + cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); + cJSON_AddStringToObject(pRoot, "pReader", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); + + if (pSender->pCurrentBlock != NULL) { + char *s; + s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + cJSON *pSnapshot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); + cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); + cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); + return pJson; +} + +char *snapshotSender2Str(SSyncSnapshotSender *pSender) { + cJSON *pJson = snapshotSender2Json(pSender); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pReceiver != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); + cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); + cJSON_AddStringToObject(pRoot, "pWriter", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pFromId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); + cJSON_AddStringToObject(pFromId, "addr", u64buf); + { + uint64_t u64 = pReceiver->fromId.addr; + cJSON *pTmp = pFromId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); + cJSON_AddItemToObject(pRoot, "fromId", pFromId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); + return pJson; +} + +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { + cJSON *pJson = snapshotReceiver2Json(pReceiver); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c new file mode 100644 index 0000000000..b12f4e76a5 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); + cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +char* syncUtilRaftId2Str(const SRaftId* p) { + cJSON* pJson = syncUtilRaftId2Json(p); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c new file mode 100644 index 0000000000..f1c26a97aa --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesGranted != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + } + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); + return pJson; +} + +char *voteGranted2Str(SVotesGranted *pVotesGranted) { + cJSON *pJson = voteGranted2Json(pVotesGranted); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesRespond != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); + } + int32_t respondNum = 0; + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); + return pJson; +} + +char *votesRespond2Str(SVotesRespond *pVotesRespond) { + cJSON *pJson = votesRespond2Json(pVotesRespond); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} From 0a7492c9c012f65e5bf1ac7677977b436ad9e87c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 12:00:18 +0800 Subject: [PATCH 45/56] ci:add the comment in cases.task --- tests/parallel_test/cases.task | 14 ++++++++++++++ tests/parallel_test/run_container.sh | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f65156f04c..ad8444f795 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1,4 +1,11 @@ +#Coulumn Define +#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand +#NA,NA,y or n,script,./test.sh -f tsim/user/basic.sim + +#unit-test ,,y,unit-test,bash test.sh + +#tsim test ,,y,script,./test.sh -f tsim/user/basic.sim ,,y,script,./test.sh -f tsim/user/password.sim ,,,script,./test.sh -f tsim/user/privilege_db.sim @@ -386,6 +393,9 @@ ,,,script,./test.sh -f tsim/tag/drop_tag.sim ,,,script,./test.sh -f tsim/tag/tbNameIn.sim ,,,script,./test.sh -f tmp/monitor.sim + +#system test + ,,,system-test,python3 ./test.py -f 0-others/taosShell.py ,,,system-test,python3 ./test.py -f 0-others/taosShellError.py ,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py @@ -982,6 +992,8 @@ ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 + +#develop test ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py @@ -993,6 +1005,8 @@ ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R + +#docs-examples test ,,,docs-examples-test,bash python.sh ,,,docs-examples-test,bash node.sh ,,,docs-examples-test,bash csharp.sh diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index 9d76f40193..eee80dad41 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -84,8 +84,8 @@ if [ $ent -ne 0 ]; then CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" - REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/:ro" - + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" + REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib:ro" else # community edition REPDIR=$WORKDIR/TDengine @@ -93,8 +93,8 @@ else CONTAINER_TESTDIR=/home/TDengine SIM_DIR=/home/TDengine/sim REP_MOUNT_PARAM="$REPDIR:/home/TDengine" - REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDengine/debug/:ro" - + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDengine/debug/" + REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib:ro" fi ulimit -c unlimited From fba45da22555f28a5ae9b703e6021c0a5a708b32 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 12:43:23 +0800 Subject: [PATCH 46/56] refact: remove assert and adjust log --- source/libs/sync/src/syncVoteMgr.c | 12 ++++++------ source/libs/sync/test/syncAppendEntriesBatchTest.cpp | 7 ------- source/libs/sync/test/syncAppendEntriesReplyTest.cpp | 6 +----- source/libs/sync/test/syncAppendEntriesTest.cpp | 6 +----- source/libs/sync/test/syncApplyMsgTest.cpp | 6 +----- .../libs/sync/test/syncConfigChangeSnapshotTest.cpp | 8 +------- source/libs/sync/test/syncConfigChangeTest.cpp | 8 +------- source/libs/sync/test/syncElectTest.cpp | 7 +------ source/libs/sync/test/syncEncodeTest.cpp | 10 ---------- source/libs/sync/test/syncEnqTest.cpp | 7 +------ source/libs/sync/test/syncEntryCacheTest.cpp | 9 --------- source/libs/sync/test/syncEntryTest.cpp | 8 -------- source/libs/sync/test/syncEnvTest.cpp | 7 +------ source/libs/sync/test/syncHashCacheTest.cpp | 8 -------- source/libs/sync/test/syncHeartbeatReplyTest.cpp | 6 +----- source/libs/sync/test/syncHeartbeatTest.cpp | 6 +----- source/libs/sync/test/syncIOClientTest.cpp | 8 +------- source/libs/sync/test/syncIOSendMsgTest.cpp | 10 ++-------- source/libs/sync/test/syncIOServerTest.cpp | 6 +----- source/libs/sync/test/syncIOTickPingTest.cpp | 5 +---- source/libs/sync/test/syncIOTickQTest.cpp | 5 +---- source/libs/sync/test/syncIndexMgrTest.cpp | 10 +--------- source/libs/sync/test/syncIndexTest.cpp | 6 +----- source/libs/sync/test/syncInitTest.cpp | 8 +------- source/libs/sync/test/syncLeaderTransferTest.cpp | 6 +----- source/libs/sync/test/syncLocalCmdTest.cpp | 6 +----- source/libs/sync/test/syncLogStoreCheck.cpp | 8 -------- source/libs/sync/test/syncLogStoreCheck2.cpp | 8 -------- source/libs/sync/test/syncLogStoreTest.cpp | 8 -------- source/libs/sync/test/syncPingReplyTest.cpp | 6 +----- source/libs/sync/test/syncPingSelfTest.cpp | 7 +------ source/libs/sync/test/syncPingTest.cpp | 6 +----- source/libs/sync/test/syncPingTimerTest.cpp | 7 +------ source/libs/sync/test/syncPingTimerTest2.cpp | 7 +------ source/libs/sync/test/syncPreSnapshotReplyTest.cpp | 6 +----- source/libs/sync/test/syncPreSnapshotTest.cpp | 6 +----- source/libs/sync/test/syncRaftCfgIndexTest.cpp | 7 +------ source/libs/sync/test/syncRaftCfgTest.cpp | 8 +------- source/libs/sync/test/syncRaftIdCheck.cpp | 5 +---- source/libs/sync/test/syncRaftLogTest.cpp | 9 +-------- source/libs/sync/test/syncRaftLogTest2.cpp | 8 -------- source/libs/sync/test/syncRaftStoreTest.cpp | 6 +----- source/libs/sync/test/syncRefTest.cpp | 6 +----- source/libs/sync/test/syncReplicateTest.cpp | 7 +------ source/libs/sync/test/syncRequestVoteReplyTest.cpp | 6 +----- source/libs/sync/test/syncRequestVoteTest.cpp | 6 +----- source/libs/sync/test/syncRestoreFromSnapshot.cpp | 9 +-------- source/libs/sync/test/syncRpcMsgTest.cpp | 6 +----- source/libs/sync/test/syncSnapshotReceiverTest.cpp | 8 +------- source/libs/sync/test/syncSnapshotRspTest.cpp | 6 +----- source/libs/sync/test/syncSnapshotSendTest.cpp | 6 +----- source/libs/sync/test/syncSnapshotSenderTest.cpp | 8 +------- source/libs/sync/test/syncSnapshotTest.cpp | 11 +---------- source/libs/sync/test/syncTest.cpp | 4 +--- source/libs/sync/test/syncTestTool.cpp | 9 +-------- source/libs/sync/test/syncTimeoutTest.cpp | 6 +----- source/libs/sync/test/syncUtilTest.cpp | 7 +------ source/libs/sync/test/syncVotesGrantedTest.cpp | 8 -------- source/libs/sync/test/syncVotesRespondTest.cpp | 8 -------- source/libs/sync/test/syncWriteTest.cpp | 10 ---------- source/libs/sync/test/sync_test_lib/inc/syncTest.h | 4 ++++ 61 files changed, 58 insertions(+), 380 deletions(-) diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index b36e5c0288..4ca4e26bec 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -62,7 +62,7 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->v void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { if (!pMsg->voteGranted) { - sNFatal(pVotesGranted->pSyncNode, " vote granted should be true"); + sNFatal(pVotesGranted->pSyncNode, "vote granted should be true"); return; } @@ -73,7 +73,7 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { } if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { - sNFatal(pVotesGranted->pSyncNode, " vote granted raftId not matched with msg"); + sNFatal(pVotesGranted->pSyncNode, "vote granted raftId not matched with msg"); return; } @@ -84,8 +84,8 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - if ((j == -1) || (j >= 0 && j < pVotesGranted->replicaNum)) { - sNFatal(pVotesGranted->pSyncNode, " invalid msg srcId, index:%d", j); + if ((j == -1) || !(j >= 0 && j < pVotesGranted->replicaNum)) { + sNFatal(pVotesGranted->pSyncNode, "invalid msg srcId, index:%d", j); return; } @@ -94,8 +94,8 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { pVotesGranted->isGranted[j] = true; } - if (pVotesGranted->votes <= pVotesGranted->replicaNum) { - sNFatal(pVotesGranted->pSyncNode, " votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + if (pVotesGranted->votes > pVotesGranted->replicaNum) { + sNFatal(pVotesGranted->pSyncNode, "votes:%d not matched with replicaNum:%d", pVotesGranted->votes, pVotesGranted->replicaNum); return; } diff --git a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp index 98b8654734..947a2cb05c 100644 --- a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp @@ -1,11 +1,4 @@ //#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncUtil.h" -#include "trpc.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index a5e96233c2..1eb803e846 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 256c13e267..7b9e04a814 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncApplyMsgTest.cpp b/source/libs/sync/test/syncApplyMsgTest.cpp index ce129015c6..a8a1931d1d 100644 --- a/source/libs/sync/test/syncApplyMsgTest.cpp +++ b/source/libs/sync/test/syncApplyMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index dcf7249d30..a6d33aa674 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index c5548d25d4..cf498933e1 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 5cdbb2cc88..59d731c823 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 216291c1d8..528cc1614d 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -1,15 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" #if 0 void logTest() { diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index bb5f968dc6..54bfb1b387 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp index 56b3d6da2e..b86422b0b1 100644 --- a/source/libs/sync/test/syncEntryCacheTest.cpp +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -1,12 +1,3 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tref.h" -#include "tskiplist.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e94755195b..369306b857 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -1,11 +1,3 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index a220c2aed4..3cd6048428 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -1,9 +1,4 @@ -#include "syncEnv.h" -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "ttime.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHashCacheTest.cpp b/source/libs/sync/test/syncHashCacheTest.cpp index 2f5bb07f9a..14a29c9a1e 100644 --- a/source/libs/sync/test/syncHashCacheTest.cpp +++ b/source/libs/sync/test/syncHashCacheTest.cpp @@ -1,12 +1,4 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "tskiplist.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp index 1fac03652b..6f7f4d18f3 100644 --- a/source/libs/sync/test/syncHeartbeatReplyTest.cpp +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp index b0c0554355..c8c38c0c38 100644 --- a/source/libs/sync/test/syncHeartbeatTest.cpp +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp index bd0221114a..f66a854657 100644 --- a/source/libs/sync/test/syncIOClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" -#include "tdatablock.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index f88e4f240d..a082f9373a 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -101,7 +95,7 @@ int main(int argc, char** argv) { syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); SEpSet epSet; - syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); + syncUtilNodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); rpcMsg.info.noResp = 1; pSyncNode->syncSendMSg(&epSet, &rpcMsg); diff --git a/source/libs/sync/test/syncIOServerTest.cpp b/source/libs/sync/test/syncIOServerTest.cpp index 1d7402e461..add7d6df60 100644 --- a/source/libs/sync/test/syncIOServerTest.cpp +++ b/source/libs/sync/test/syncIOServerTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 9c2342828e..71274f5ca3 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 64b65f25c8..9ef8a7ad40 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 23b8693a0b..f84119045f 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -1,12 +1,4 @@ -#include "syncIndexMgr.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexTest.cpp b/source/libs/sync/test/syncIndexTest.cpp index 763117c0c9..a44d9c4e31 100644 --- a/source/libs/sync/test/syncIndexTest.cpp +++ b/source/libs/sync/test/syncIndexTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void print(SHashObj *pNextIndex) { printf("----------------\n"); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 2fe5dd7e18..b15a767db5 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLeaderTransferTest.cpp b/source/libs/sync/test/syncLeaderTransferTest.cpp index bd1f22edff..fd8cdfaac0 100644 --- a/source/libs/sync/test/syncLeaderTransferTest.cpp +++ b/source/libs/sync/test/syncLeaderTransferTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp index b42626df29..fa0d91d45f 100644 --- a/source/libs/sync/test/syncLocalCmdTest.cpp +++ b/source/libs/sync/test/syncLocalCmdTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 0161160a75..e02c484729 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp index 29ad0610e7..fce0a21a47 100644 --- a/source/libs/sync/test/syncLogStoreCheck2.cpp +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -1,12 +1,4 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 832b42bf80..1b898803ef 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index e5158a7932..661509e9a2 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 781ffc31db..19f377e037 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 1777b9b9e2..54cc506b03 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 5a0b5e9953..ded7e22eac 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 09f56815cd..4f1bb64798 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp index a30dcc2c54..e5175f1101 100644 --- a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp @@ -1,10 +1,6 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotTest.cpp b/source/libs/sync/test/syncPreSnapshotTest.cpp index 03894dfa84..2ce95abea3 100644 --- a/source/libs/sync/test/syncPreSnapshotTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgIndexTest.cpp b/source/libs/sync/test/syncRaftCfgIndexTest.cpp index 522da159b5..e6d3f23b58 100644 --- a/source/libs/sync/test/syncRaftCfgIndexTest.cpp +++ b/source/libs/sync/test/syncRaftCfgIndexTest.cpp @@ -1,10 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index b1f820c78e..c841a68fde 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -1,10 +1,4 @@ -#include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index e7ef69da20..08716c876b 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void usage(char* exe) { printf("Usage: %s host port \n", exe); diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp index 278113919a..e309a2e432 100644 --- a/source/libs/sync/test/syncRaftLogTest.cpp +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -1,12 +1,5 @@ #include "syncRaftLog.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 3d50b63ff9..a7752dcb8b 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 070f8c0f07..87798a7d80 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,9 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index c5132018a2..47bc290abe 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "tref.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 7506ae19bd..2419ec0974 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 3d9db17725..973ccf040c 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 94e1504add..275f7804bf 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRestoreFromSnapshot.cpp b/source/libs/sync/test/syncRestoreFromSnapshot.cpp index 470dd678b0..44444a7f5e 100644 --- a/source/libs/sync/test/syncRestoreFromSnapshot.cpp +++ b/source/libs/sync/test/syncRestoreFromSnapshot.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 127d8e1c41..3f66cd9d1c 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 6eb8548740..49b06a7d1b 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 63905c2182..0a06b512ea 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 83f1dfebb3..46f33e0c56 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index 7b2c13463c..9a234d412e 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index e2264fd08c..40eac0b55f 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -1,14 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 97de81572a..55480802da 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" /* typedef enum { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index e6f6006410..b0fe612b7f 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -1,12 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index bae5e0ea88..ed0742f499 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 411915fe22..bbd4fa7d18 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -1,9 +1,4 @@ -#include "syncUtil.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 6a8404308b..2d2a12c656 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -1,11 +1,3 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index d5e7d71030..712e97edc9 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -1,14 +1,6 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" #include "syncTest.h" - void logTest() { sTrace("--- sync log test: trace"); sDebug("--- sync log test: debug"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 2e5c26719b..56dd0a5845 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -1,15 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h index 241a4a5e02..a1881bf1b3 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncTest.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -22,8 +22,10 @@ extern "C" { #include "syncInt.h" +#include "tref.h" #include "wal.h" +#include "tref.h" #include "syncEnv.h" #include "syncIO.h" #include "syncIndexMgr.h" @@ -38,6 +40,8 @@ extern "C" { #include "syncUtil.h" #include "syncVoteMgr.h" +extern void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); + cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); char* syncEntry2Str(const SSyncRaftEntry* pEntry); void syncEntryPrint(const SSyncRaftEntry* pObj); From c09d7ed8411bfea34aa38df03686e0ed61a026a4 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 12:46:10 +0800 Subject: [PATCH 47/56] ci:add the env that builds with sanitizer --- tests/parallel_test/run_container.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index eee80dad41..b5deb09341 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -125,6 +125,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ -v $REP_MOUNT_DEBUG \ + -v $REP_MOUNT_LIB \ -v $MOUNT_DIR \ -v ${SOURCEDIR}:/usr/local/src/ \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ From 6f7d43c3715f1da83425c6fe86e5752e03ad5eaf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Nov 2022 13:33:55 +0800 Subject: [PATCH 48/56] fix: case when type error --- source/libs/parser/src/parTranslater.c | 41 +++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f7dce1637a..05fc651618 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1252,6 +1252,19 @@ static bool dataTypeEqual(const SDataType* l, const SDataType* r) { return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale); } +// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail +static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type != r->type) { + return 1; + } + + if (l->bytes != r->bytes) { + return l->bytes > r->bytes ? 1 : -1; + } + + return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; +} + static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); @@ -1876,10 +1889,15 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW } pWhenThen->pWhen = pIsTrue; } - if (first) { - first = false; + if (first || dataTypeComp(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType) < 0) { pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; - } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { + } + first = false; + } + + FOREACH(pNode, pCaseWhen->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pWhenThen->pThen, pCaseWhen->node.resType, &pCastFunc); if (TSDB_CODE_SUCCESS != pCxt->errCode) { @@ -1889,6 +1907,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->node.resType = pCaseWhen->node.resType; } } + if (NULL != pCaseWhen->pElse && !dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pCaseWhen->pElse)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pCaseWhen->pElse, pCaseWhen->node.resType, &pCastFunc); @@ -3431,19 +3450,6 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return (SNode*)pCol; } -// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail -static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { - if (l->type != r->type) { - return 1; - } - - if (l->bytes != r->bytes) { - return l->bytes > r->bytes ? 1 : -1; - } - - return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; -} - static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) { SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); @@ -4969,7 +4975,8 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameExtractFullName(&name, usedbReq.db); - int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); + int32_t code = + getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); } From cebc6b66973693c07eeed7ca19db9b00e2fe932e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 10 Nov 2022 14:43:37 +0800 Subject: [PATCH 49/56] fix(stream): state and session agg partition tbname --- source/libs/executor/src/timewindowoperator.c | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7b6ed5b67d..46e87527f1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3843,7 +3843,7 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { return TSDB_CODE_SUCCESS; } -void doBuildDeleteDataBlock(SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { +void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { blockDataCleanup(pBlock); int32_t size = tSimpleHashGetSize(pStDeleted); if (size == 0) { @@ -3869,6 +3869,26 @@ void doBuildDeleteDataBlock(SSHashObj* pStDeleted, SSDataBlock* pBlock, void** I colDataAppendNULL(pCalStCol, pBlock->info.rows); SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataAppendNULL(pCalEdCol, pBlock->info.rows); + + SHashObj* pGroupIdTbNameMap; + if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + SStreamSessionAggOperatorInfo* pInfo = pOp->info; + pGroupIdTbNameMap = pInfo->pGroupIdTbNameMap; + } else if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + SStreamStateAggOperatorInfo* pInfo = pOp->info; + pGroupIdTbNameMap = pInfo->pGroupIdTbNameMap; + } + + char* tbname = taosHashGet(pGroupIdTbNameMap, &res->groupId, sizeof(int64_t)); + SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + if (tbname == NULL) { + colDataAppendNULL(pTableCol, pBlock->info.rows); + } else { + char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; + STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); + colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + } pBlock->info.rows += 1; } if ((*Ite) == NULL) { @@ -4013,7 +4033,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { - doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; @@ -4112,7 +4132,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; @@ -4238,7 +4258,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "semi session delete"); return pInfo->pDelRes; @@ -4312,7 +4332,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "semi session delete"); return pInfo->pDelRes; @@ -4563,7 +4583,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; int64_t maxTs = INT64_MIN; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "single state delete"); return pInfo->pDelRes; @@ -4630,7 +4650,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "single state delete"); return pInfo->pDelRes; From 82fb1878ed268685f120cbc21a22ae06d0ab064a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 15:13:14 +0800 Subject: [PATCH 50/56] ci:add the env that builds with sanitizer --- tests/parallel_test/container_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 4e06643ac4..f942d554ff 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -54,7 +54,7 @@ fi date docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT || exit 1" if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" @@ -69,7 +69,7 @@ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT || exit " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan From f1a3050a581d2be51e3b361a2a8577911f8205c6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 15:15:13 +0800 Subject: [PATCH 51/56] test: enable asan test mode --- tests/script/sh/checkAsan.sh | 39 ++++++++++++++++++++++++++++++++++++ tests/script/sh/exec.sh | 4 ++-- tests/script/test.sh | 9 +++++++-- 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100755 tests/script/sh/checkAsan.sh diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh new file mode 100755 index 0000000000..d4801891ed --- /dev/null +++ b/tests/script/sh/checkAsan.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set +e +#set -x + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +LOG_DIR=$TAOS_DIR/sim/tsim/asan + +error_num=`cat ${LOG_DIR}/tsim.asan | grep "ERROR" | wc -l` +memory_leak=`cat ${LOG_DIR}/tsim.asan | grep "Direct leak" | wc -l` +indirect_leak=`cat ${LOG_DIR}/tsim.asan | grep "Indirect leak" | wc -l` +runtime_error=`cat ${LOG_DIR}/tsim.asan | grep "runtime error" | wc -l` + +echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" +echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" +echo -e "\033[44;32;1m"asan indirect_leak: $indirect_leak"\033[0m" +echo -e "\033[44;32;1m"asan runtime error: $runtime_error"\033[0m" + +let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error" + +if [ $errors -eq 0 ]; then + echo -e "\033[44;32;1m"no asan errors"\033[0m" + exit 0 +else + echo -e "\033[44;31;1m"asan total errors: $errors"\033[0m" + cat ${LOG_DIR}/tsim.asan + exit 1 +fi \ No newline at end of file diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 74015eebd6..3f2c5d268c 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -80,7 +80,7 @@ LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data MGMT_DIR=$NODE_DIR/data/mgmt TSDB_DIR=$NODE_DIR/data/tsdb - +ASAN_DIR=$SIM_DIR/tsim/asan TAOS_CFG=$NODE_DIR/cfg/taos.cfg echo ------------ $EXEC_OPTON $NODE_NAME @@ -105,7 +105,7 @@ if [ "$EXEC_OPTON" = "start" ]; then nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else echo "nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" - nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2> $ASAN_DIR/$NODE_NAME.asan & fi else diff --git a/tests/script/test.sh b/tests/script/test.sh index b38d331715..ee6f6f6ab7 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -74,6 +74,7 @@ PRG_DIR=$SIM_DIR/tsim CFG_DIR=$PRG_DIR/cfg LOG_DIR=$PRG_DIR/log DATA_DIR=$PRG_DIR/data +ASAN_DIR=$PRG_DIR/asan chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" @@ -82,14 +83,17 @@ echo "BUILD_DIR: $BUILD_DIR" echo "SIM_DIR : $SIM_DIR" echo "CODE_DIR : $CODE_DIR" echo "CFG_DIR : $CFG_DIR" +echo "ASAN_DIR : $ASAN_DIR" rm -rf $SIM_DIR/* rm -rf $LOG_DIR rm -rf $CFG_DIR +rm -rf $ASAN_DIR mkdir -p $PRG_DIR mkdir -p $LOG_DIR mkdir -p $CFG_DIR +mkdir -p $ASAN_DIR TAOS_CFG=$PRG_DIR/cfg/taos.cfg touch -f $TAOS_CFG @@ -132,8 +136,9 @@ if [ -n "$FILE_NAME" ]; then echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME -v $PROGRAM -c $CFG_DIR -f $FILE_NAME -v else - echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME - $PROGRAM -c $CFG_DIR -f $FILE_NAME + echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME + $PROGRAM -c $CFG_DIR -f $FILE_NAME 2> $ASAN_DIR/tsim.asan + $TOP_DIR/tests/script/sh/checkAsan.sh fi else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim From d0c317cfa9ee3106350bfacfdf5fc090f5099ffa Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 15:16:42 +0800 Subject: [PATCH 52/56] ci:add the env that builds with sanitizer --- tests/parallel_test/container_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index f942d554ff..7c18f5cab5 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -69,7 +69,7 @@ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT || exit " + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT || exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan From 7532c1d9f9cc89e6209165d980e78402b7f70881 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 15:32:37 +0800 Subject: [PATCH 53/56] test: enable asan test mode --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ad8444f795..6775760ee4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,8 +6,8 @@ ,,y,unit-test,bash test.sh #tsim test -,,y,script,./test.sh -f tsim/user/basic.sim -,,y,script,./test.sh -f tsim/user/password.sim +,,,script,./test.sh -f tsim/user/basic.sim +,,,script,./test.sh -f tsim/user/password.sim ,,,script,./test.sh -f tsim/user/privilege_db.sim ,,,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,,script,./test.sh -f tsim/db/alter_option.sim From b7aff29d2ea46d4da3ce9b0e0f0b5dbff6bbd0b1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 10 Nov 2022 16:32:05 +0800 Subject: [PATCH 54/56] test:add logs of starting taosd in taosdShell.py --- tests/system-test/0-others/taosdShell.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index b743783a4f..581448a6d9 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -2,6 +2,7 @@ import taos import sys import time +from datetime import datetime import socket import os import platform @@ -100,8 +101,11 @@ class TDTestCase: processName="taosd" taosdCmd = taosdCmdRun + startAction tdLog.printNoPrefix("%s"%taosdCmd) - os.system(f"nohup {taosdCmd} & ") + logTime=datetime.now().strftime('%Y%m%d_%H%M%S_%f') + os.system(f"nohup {taosdCmd} > {logTime}.log 2>&1 & ") self.checkAndstopPro(processName,startAction) + os.system(f"rm -rf {logTime}.log") + def taosdCommandExe(self,startAction,taosdCmdRun): taosdCmd = taosdCmdRun + startAction @@ -207,7 +211,7 @@ class TDTestCase: os.system(" mkdir -p taosdCaseTmp ") os.system("echo \'TAOS_QUERY_POLICY=3\' > taosdCaseTmp/.env ") self.taosdCommandStop(startAction,taosdCmdRun) - os.system(" rm -rf taosdCaseTmp/.env ") + os.system(" rm -rf taosdCaseTmp ") startAction = " -V" tdLog.printNoPrefix("================================ parameter: %s"%startAction) From 64f22d560c1fc4d24a7882aa37222dc65d82389e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 10 Nov 2022 16:36:49 +0800 Subject: [PATCH 55/56] refactor(stream): drop stream task --- source/libs/stream/src/streamMeta.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 682bfbaf8d..da082b7f74 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -187,11 +187,9 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { while (1) { int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING); - if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { + if (schedStatus != TASK_SCHED_STATUS__ACTIVE) { tFreeSStreamTask(pTask); break; - } else if (schedStatus == TASK_SCHED_STATUS__DROPPING) { - break; } taosMsleep(10); } From c8db4da289b43e64a3b40a33297716d17889cdaa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 16:56:15 +0800 Subject: [PATCH 56/56] fix: asan script error --- tests/script/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/test.sh b/tests/script/test.sh index ee6f6f6ab7..1e3319dd0a 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -138,7 +138,7 @@ if [ -n "$FILE_NAME" ]; then else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME $PROGRAM -c $CFG_DIR -f $FILE_NAME 2> $ASAN_DIR/tsim.asan - $TOP_DIR/tests/script/sh/checkAsan.sh + $CODE_DIR/sh/checkAsan.sh fi else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim